Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Staged FMC pipeline

Aaron Cheriand6af8a58361f
diff --git a/freemocap/__init__.py b/freemocap/__init__.py
new file mode 100644
index 00000000..ef4b4f30
--- /dev/null
+++ b/freemocap/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Mar 1 13:39:34 2021
+
+@author: Rontc
+"""
+
diff --git a/freemocap/calibrate.py b/freemocap/calibrate.py
new file mode 100644
index 00000000..eae7dfc2
--- /dev/null
+++ b/freemocap/calibrate.py
@@ -0,0 +1,200 @@
+from aniposelib.cameras import CameraGroup
+from freemocap import stolenfromanipose
+
+from aniposelib.utils import load_pose2d_fnames
+import numpy as np
+import cv2
+
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+from freemocap import reconstruct3D, stolenfromanipose
+import os
+from rich.progress import track
+
+
+
+def CalibrateCaptureVolume(session,board):
+
+ calVideoFrameLength = 60
+ createCalibrationVideos(session,calVideoFrameLength)
+
+ vidnames = []
+ cam_names = []
+ for count,thisVidPath in enumerate(session.calVidPath.glob('*.mp4'),start=1):
+ vidnames.append([str(thisVidPath)])
+ cam_names.append(str(count))
+ session.numCams = count
+
+ cgroup = stolenfromanipose.CameraGroup.from_names(cam_names, fisheye=True) #Looking through their code... it looks lke the 'fisheye=True' doesn't do much (see 2020-03-29 obsidian note)
+
+
+
+ calibrationFile = '{}_calibration.yaml'.format(session.sessionID)
+
+ session.cameraCalFilePath = session.sessionPath/calibrationFile
+
+ forceRunCameraExtrinsicsCalibration = True #set this to True to re-run the camera calibration and saveover the existing config file
+
+ # if not session.cameraCalFilePath.exists() or forceRunCameraExtrinsicsCalibration: #if that config file doesn't exist, run the camera calibration whosists. If it does, then just load the toml
+ ## this will take a few minutes
+ ## it will detect the charuco board in the videos,
+ ## then calibrate the cameras based on the detections, using iterative bundle adjustment
+ # error,all_rows,merged = cgroup.calibrate_videos(vidnames, board) #JSM NOTE - in its original form, this method doesn't throw an error when a video fails to load. This is an easy fix!
+
+ #%% This next bit is a copy-paste of the inner bits of `cgroup.calibrate_videos` - Part 1 of 2
+ """Takes as input a list of list of video filenames, one list of each camera.
+ Also takes a board which specifies what should be detected in the videos"""
+
+ all_rows = cgroup.get_rows_videos(vidnames, board, verbose=True)
+
+ cgroup.set_camera_sizes_videos(vidnames)
+
+ if session.debug:
+ fig = plt.figure(47290)
+ for camNum in range(len(all_rows)):
+ ax = fig.add_subplot(2,2,camNum+1)
+ # cap = cv2.VideoCapture(vidnames[camNum][0]) #this is crazy inefficient - we should save out the first image somewhere and save it in an accessible location
+ # ret, frame = cap.read()
+ # plt.imshow(frame)
+ # cap.release()
+ for frNum in range(len(all_rows[camNum])):
+ corners = np.squeeze(all_rows[camNum][frNum]['filled'])
+ plt.plot(corners[:,0], corners[:,1],'.-')
+ # xmin = np.nanmin(corners[:,0])*.9
+ # xmax = np.nanmax(corners[:,0])*1.1
+ # ymin = np.nanmin(corners[:,1])*.9
+ # ymax = np.nanmax(corners[:,1])*1.1
+ # ax.set_xlim(xmin, xmax)
+ # ax.set_xlim(ymin, ymax)
+ plt.show()
+
+
+
+#%% This next bit is a copy-paste of the inner bits of `cgroup.calibrate_videos` - Part 2 of 2
+ #error, merged = cgroup.calibrate_rows(all_rows, board,
+ # init_intrinsics=True,
+ # init_extrinsics=True)
+#%% End of the copy-paste of the inner bits of `cgroup.calibrate_videos` - END
+
+ #JSM NOTE - need add method to extract Charuco board points from the calibrate_videos pipeline
+
+ ## if you need to save and load
+ ## example saving and loading for later
+ error,merged = cgroup.calibrate_videos(vidnames, board)
+ cgroup.dump(session.cameraCalFilePath) #JSM NOTE - let's just use .yaml's unless there is some reason to use .toml
+ mergename = session.sessionPath/'merged.npy'
+ np.save(mergename,merged)
+ else:
+ cgroup = CameraGroup.load(session.cameraCalFilePath) #load previous calibration config
+
+ session.cgroup = cgroup
+ n_frames= 40
+ startframe = 0
+ n_trackedPoints = 24
+ framelist = range(startframe,startframe+n_frames)
+
+ charucoarray = np.empty([session.numCams,n_frames,n_trackedPoints,1,2])
+ charucoarray[:] = np.nan
+
+ data = np.load(session.sessionPath/'merged.npy',allow_pickle = True)
+
+ for cam in range(session.numCams):
+ for count,frame in enumerate(framelist):
+ try:
+ charucoarray[cam][count] = data[frame][cam]['filled']
+ except:
+ print('failed frame:', frame)
+ continue
+ charuco_nCams_nFrames_nImgPts_XY = np.array(charucoarray)
+
+ session.charuco_nCams_nFrames_nImgPts_XY = charuco_nCams_nFrames_nImgPts_XY
+
+ charuco_params = charuco_nCams_nFrames_nImgPts_XY.shape[0:3]
+
+ charuco_fr_mar_dim = reconstruct3D.reconstruct3D(session,charuco_nCams_nFrames_nImgPts_XY,charuco_params)
+
+ mean_charuco_fr_mar_dim = np.nanmean(charuco_fr_mar_dim,axis = 0)
+
+ # charry_reshaped = charucoarray.reshape(session.numCams, -1, 2)
+
+ # char_flat = cgroup.triangulate(charry_reshaped, progress=True)
+ # charReprojerr_flat = cgroup.reprojection_error(char_flat, charry_reshaped, mean=True)
+
+ # char_fr_mar_dim = char_flat.reshape(n_frames, n_trackedPoints, 3)
+ # charReprojErr_fr_mar_err = charReprojerr_flat.reshape(n_frames, n_trackedPoints)
+
+
+ if session.debug:
+ fig = plt.figure()
+ #mean charuco position
+ ax1 = fig.add_subplot(111,projection='3d')
+ ax1.cla()
+ x = mean_charuco_fr_mar_dim[:][:,0]
+ y = mean_charuco_fr_mar_dim[:][:,1]
+ z = mean_charuco_fr_mar_dim[:][:,2]
+ mx = np.nanmean(x)
+ my = np.nanmean(y)
+ mz = np.nanmean(z)
+ ax1.scatter(x,y,z, marker='o')
+ ax1.set_title('Mean Charuco Point Positions')
+
+ ax1.set_xlabel('x')
+ ax1.set_ylabel('y')
+ ax1.set_zlabel('z')
+
+ for camNum in range(len(cgroup.cameras)):
+ np.append(mx, cgroup.cameras[camNum].tvec[0])
+ np.append(my, cgroup.cameras[camNum].tvec[1])
+ np.append(mz, cgroup.cameras[camNum].tvec[2])
+
+ ax1.scatter(cgroup.cameras[camNum].tvec[0], cgroup.cameras[camNum].tvec[1], cgroup.cameras[camNum].tvec[2], marker = 'p')
+
+
+ mx = np.nanmean(mx)
+ my = np.nanmean(my)
+ mz = np.nanmean(mz)
+
+ ax1.set_xlim(mx-100,mx+100)
+ ax1.set_ylim(my-100,my+100)
+ ax1.set_zlim(mz-100,mz+100)
+
+ # #charuco points over time
+ # ax2 = fig.add_subplot(122)
+ # ax2.cla()
+ # numPts = charuco_fr_mar_dim.shape[1]
+ # for pp in range(numPts):
+ # ax2.plot(charuco_fr_mar_dim[:,pp,:])
+ # ax2.set_title('Charuco Point positions over time')
+ plt.show()
+
+ path_to_charuco_array = session.dataArrayPath/'charuco_points.npy'
+ np.save(path_to_charuco_array, mean_charuco_fr_mar_dim)
+ return cgroup,mean_charuco_fr_mar_dim
+
+def createCalibrationVideos(session,calVideoFrameLength):
+ vidList = os.listdir(session.syncedVidPath)
+ framelist = list(range(calVideoFrameLength))
+ codec = 'DIVX'
+ for count,vid in enumerate(vidList,start=1):
+ cam_name = "Cam{}".format(count)
+ cap = cv2.VideoCapture(str(session.syncedVidPath/vid))
+ fourcc = cv2.VideoWriter_fourcc(*codec)
+
+ #grab resolution parameters from the videos
+ resWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ resHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ framerate = int(cap.get(cv2.CAP_PROP_FPS))
+
+ saveName = session.sessionID +'_trimmed_' + cam_name + '.mp4' #create a name for the trimmed video
+ saveCalVidPath = str(session.calVidPath/saveName) #create an output path for the function
+
+ success, image = cap.read() #start reading frames
+
+ out = cv2.VideoWriter(saveCalVidPath, fourcc, framerate, (resWidth,resHeight))
+ print('Trimming ' + cam_name)
+ for frame in track(framelist):
+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need
+ success, image = cap.read()
+ out.write(image)
+ cap.release()
+ out.release()
\ No newline at end of file
diff --git a/freemocap/createvideo.py b/freemocap/createvideo.py
new file mode 100644
index 00000000..3ae52d2a
--- /dev/null
+++ b/freemocap/createvideo.py
@@ -0,0 +1,28 @@
+import glob
+import cv2
+from pathlib import Path
+import os
+
+
+def createVideo(session):
+
+ vidSavePath = str(session.sessionPath/'{}_outVid.mp4'.format(session.sessionID))
+ fps = 30
+ shape = 1078,647
+ frame_array = []
+
+ for filename in session.imOutPath.glob('*.png'):
+ filename= str(filename)
+
+ #reading each files
+ img = cv2.imread(filename)
+ #resized = cv2.resize(img,shape)
+ #shape = img.shape[:2]
+ resized = cv2.resize(img,shape)
+ #inserting the frames into an image array
+ frame_array.append(resized)
+ out = cv2.VideoWriter(vidSavePath,cv2.VideoWriter_fourcc(*'DIVX'), fps, shape)
+ for i in range(len(frame_array)):
+ # writing to a image array
+ out.write(frame_array[i])
+ out.release()
\ No newline at end of file
diff --git a/freemocap/dlc.py b/freemocap/dlc.py
new file mode 100644
index 00000000..bd002983
--- /dev/null
+++ b/freemocap/dlc.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Feb 26 12:42:08 2021
+
+@author: Rontc
+"""
+import glob
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+
+
+def ParseDLCdata(sesh): #note this is all sloppy af just to get a completed video together. Latter iterations will use DLC methods and be smarter about Pandas Dataframs and whatnot
+
+
+ dlcCSVPaths = sesh.dlcDataPath.glob('*.csv') #NOTE - Super hacky here. Need to fix these methods (replace with DLC native functions?)
+
+ numCams = int(sesh.numCams)
+ numFrames = sesh.numFrames+1
+
+ if sesh.sessionID == 'test6_01_21a': #NOTE - THIS IS DUMB AND BAD
+ numCols = 8 #this is the number of tracked points in DLC - hardcoding for now because I am a bad person
+ else:
+ numCols = 3 #NOTE - AGAIN< THIS IS DUMB AND BAD >
+
+ dlc_nCams_nFrames_nImgPts_XYC = np.ndarray([numCams, numFrames, numCols, 3])
+ dlc_nCams_nFrames_nImgPts_XYC.fill(np.nan)
+ camNum = -1
+ for thisCSVpath in dlcCSVPaths: #triple nested for loop so I don't have to figure out np.reshape lol
+ camNum += 1
+ thisCam_dlcDataFrame = pd.read_csv(thisCSVpath, skiprows=1, header = [0,1]) #NOTE - This is dumb and bad. No need for this pandas dataframe, I think
+ thisCam_dlcNumpy = thisCam_dlcDataFrame.to_numpy()
+ thisCam_dlcNumpy = thisCam_dlcNumpy[:, 1:] #remove first column (frame numbers)
+
+ for thisFrame in range(thisCam_dlcNumpy.shape[0]):
+ for thisImgPt in range(0,thisCam_dlcNumpy.shape[1],3):
+ dlc_nCams_nFrames_nImgPts_XYC[camNum,thisFrame,int(thisImgPt/3), 0] = thisCam_dlcNumpy[thisFrame, thisImgPt]
+ dlc_nCams_nFrames_nImgPts_XYC[camNum,thisFrame,int(thisImgPt/3), 1] = thisCam_dlcNumpy[thisFrame, thisImgPt+1]
+ dlc_nCams_nFrames_nImgPts_XYC[camNum,thisFrame,int(thisImgPt/3), 2] = thisCam_dlcNumpy[thisFrame, thisImgPt+2]
+
+
+ if sesh.debug:
+ fig = plt.figure()
+
+ ax1 = fig.add_subplot(221)
+ ax2 = fig.add_subplot(222)
+ ax3 = fig.add_subplot(223)
+ ax4 = fig.add_subplot(224)
+
+ cam1im = sesh.firstImage_nCams_list[0]
+ cam2im = sesh.firstImage_nCams_list[1]
+ cam3im = sesh.firstImage_nCams_list[2]
+ cam4im = sesh.firstImage_nCams_list[3]
+
+ ax1.imshow(cam1im)
+ ax1.plot(dlc_nCams_nFrames_nImgPts_XYC[0,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[0,0,:,1])
+
+ ax2.imshow(cam2im)
+ ax2.plot(dlc_nCams_nFrames_nImgPts_XYC[1,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[1,0,:,1])
+
+ ax3.imshow(cam3im)
+ ax3.plot(dlc_nCams_nFrames_nImgPts_XYC[2,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[2,0,:,1])
+
+ ax4.imshow(cam4im)
+ ax4.plot(dlc_nCams_nFrames_nImgPts_XYC[3,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[3,0,:,1])
+
+ plt.show()
+
+ sesh.dlc_nCams_nFrames_nImgPts_XYC = dlc_nCams_nFrames_nImgPts_XYC
\ No newline at end of file
diff --git a/freemocap/initialization.py b/freemocap/initialization.py
new file mode 100644
index 00000000..b787b427
--- /dev/null
+++ b/freemocap/initialization.py
@@ -0,0 +1,180 @@
+
+from pathlib import Path
+from ruamel.yaml import YAML
+import datetime
+import sys
+import pandas as pd
+import numpy as np
+
+from freemocap.webcam import recordGUI,camsetup
+from freemocap import recordingconfig, stolenfromanipose
+
+def initialize(session, stage, board):
+
+ print('Starting initialization for stage {}'.format(stage))
+ #if stage == 1:
+ filepath = Path.cwd()
+# %% Stage One Initialization
+ if stage == 1:
+
+
+ #create session ID
+ sessionID_in = datetime.datetime.now().strftime("sesh_%y-%m-%d_%H%M%S")
+ session.sessionID = sessionID_in
+
+ #load the parameters YAML and extract the saved parameters if possible, and the default otherwise
+ parameter_path = filepath/'parameters.yaml'
+ parameters_yaml = YAML()
+ parameters = parameters_yaml.load(parameter_path)
+
+ try:
+ rotation_entry = parameters['saved']['rotations']
+ parameter_entry = parameters['saved']['parameters']
+ except:
+ print("Using Default Config")
+ rotation_entry = parameters['default']['rotations']
+ parameter_entry = parameters['default']['parameters']
+
+ proceedToRecording = False #create this boolean, set it to false, and if the user wants to record
+ #later in the pipeline, it will be set to true
+
+ #run the GUI to get the tasks, the cams chosen, the camera settings, and the session ID
+ task, cam_inputs, rotDict, paramDict, session.sessionID = recordGUI.RunGUI(sessionID_in,rotation_entry,parameter_entry)
+
+ #update the saved parameters in the YAML
+ parameters['saved']['rotations'] = rotDict
+ parameters['saved']['parameters'] = paramDict
+ parameters_yaml.dump(parameters, parameter_path)
+
+ #create a list from the rotation dictionary to be used in running webcams
+ rotation_input = list(rotDict.values())
+
+ if task == 'setup':
+ #run setup processes, and then check if th user wants to proceed to recording
+ camsetup.RunSetup(cam_inputs,rotation_input,paramDict)
+ proceedToRecording,session.sessionID = recordGUI.RunProceedtoRecordGUI(sessionID_in)
+
+ elif task == 'record':
+ proceedToRecording = True
+
+ if proceedToRecording:
+ #create these session properties to be used later in the pipeline
+ session.cam_inputs = cam_inputs
+ session.parameterDictionary = paramDict
+ session.rotationInputs = rotation_input
+
+ #create a config yaml and text file for this session
+ config_yaml_path = recordingconfig.createSession(session,filepath)
+ recordingconfig.createSessionTxt(session,paramDict,rotDict)
+
+ print('Proceeding to Stage One')
+
+ else:
+
+ sys.exit('Recording Cancelled')
+# %% Stage Two Initalization
+ if stage == 2:
+ session.sessionPath = filepath/'Data'/session.sessionID #create a session path based on the sessionID
+
+ #load the config yaml for this session, and add all the paths to the session file
+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID)
+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class
+
+ #from the config settings add the camera input parameters to parameter dictionary
+ session.parameterDictionary = config_settings['CamInputs']['ParameterDict']
+ session.rotationInputs = config_settings['CamInputs']['RotationInputs']
+
+ #initialize the path to the timestamp csv
+ csvName = session.sessionID + '.csv'
+ csvPath = session.sessionPath/csvName
+
+ #read CSV data, turn it into a data frame
+ timeStampData = pd.read_csv (csvPath)
+ timeStampData = timeStampData.iloc[:,1:]
+
+ #get the camIDs and number of cameras (numCamRange) from the dataframe
+ camIDs = list(timeStampData.columns)
+ numCamRange = range(len(camIDs))
+
+ #create names for each of the raw videos
+ vidNames = []
+ for x in numCamRange:
+ singleVidName = 'raw_cam{}.mp4'.format(x+1)
+ vidNames.append(singleVidName)
+
+
+ #initialize all the session variables we'll need to run the rest of the pipeline
+ session.timeStampData = timeStampData
+ session.camIDs = camIDs
+ session.numCamRange = numCamRange
+ session.vidNames = vidNames
+
+ if stage == 3:
+
+ session.sessionPath = filepath/'Data'/session.sessionID #create a session path based on the sessionID
+
+ #load the config yaml for this session, and add all the paths to the session file
+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID)
+
+ if session.yamlPath.is_file(): #if the config yaml exists (from a webcam recording)
+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class
+ else: #if it doesn't exist (because of a GoPro/external camera recording)
+ recordingconfig.createSession(session,filepath)
+
+ if stage == 4:
+
+ session.sessionPath = filepath/'Data'/session.sessionID
+
+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml
+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class
+
+ if stage == 5:
+
+ session.sessionPath = filepath/'Data'/session.sessionID
+
+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml
+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class
+
+ session.openPose_jsonPathList = config_settings['openPose_jsonPathList']
+ session.numCams = len(session.openPose_jsonPathList)
+
+ if stage == 6:
+
+ session.sessionPath = filepath/'Data'/session.sessionID
+
+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml
+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class
+
+ session.openPoseData_nCams_nFrames_nImgPts_XY = np.load(session.dataArrayPath/'openPoseData_nCams_nFrames_nImgPts_XY.npy')
+ calibrationFile = '{}_calibration.yaml'.format(session.sessionID)
+ session.cameraConfigFilePath = session.sessionPath/calibrationFile
+ session.cgroup = stolenfromanipose.CameraGroup.load(session.cameraConfigFilePath)
+
+ if stage == 7:
+
+ session.sessionPath = filepath/'Data'/session.sessionID
+
+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml
+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class
+
+ session.openPose_imgPathList = config_settings['openPose_imgPathList']
+ session.numCams = len(session.openPose_imgPathList)
+
+ session.mean_charuco_fr_mar_dim = np.load(session.dataArrayPath/'charuco_points.npy')
+ session.skel_fr_mar_dim = np.load(session.dataArrayPath/'skeleton_points.npy')
+
+ if stage == 8:
+
+ session.sessionPath = filepath/'Data'/session.sessionID
+
+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml
+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class
+
+# %%
diff --git a/freemocap/playskeleton.py b/freemocap/playskeleton.py
new file mode 100644
index 00000000..757aef3b
--- /dev/null
+++ b/freemocap/playskeleton.py
@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Feb 26 12:51:21 2021
+
+@author: Rontc
+"""
+
+import numpy as np
+#from mpl_toolkits.mplot3d import Axes3D
+import matplotlib.pyplot as plt
+import matplotlib as mpl
+import cv2
+from pathlib import Path
+#from glob import glob
+
+# def PlaySkeleton(session, vidType):
+
+# # vidType = 0 #No Cam Ims
+# # vidType = 1 #Only Cam1 im
+# # vidType = 2 #All Cam Ims
+
+# #where to put the output images (later will be justa video)
+
+
+# #os.chdir(imOutPath)
+
+
+
+
+# #set axis limits based on charuco board
+# # mx = np.mean(charuco_x)
+# # my = np.mean(charuco_y)
+# # mz = np.mean(charuco_z)
+
+# #set up figure
+# # fig = plt.figure(figsize=([11,8]))
+
+# # if vidType==0:
+# # axMain = fig.add_subplot(111,projection='3d')
+# # elif vidType == 1:
+# # axMain = fig.add_subplot(121,projection='3d')
+# # axCam1 = fig.add_subplot(122)
+# # elif vidType == 2:
+# # figGridSpec = fig.add_gridspec(2,4)
+# # axMain = fig.add_subplot(figGridSpec[:2,:2],projection='3d')
+# # axCam1 = fig.add_subplot(figGridSpec[0,2])
+# # axCam2 = fig.add_subplot(figGridSpec[0,3])
+# # axCam3 = fig.add_subplot(figGridSpec[1,2])
+# # axCam4 = fig.add_subplot(figGridSpec[1,3])
+
+
+
+
+# ##############################################################################################
+# ##Frame-by-frame animation loop starts here
+# ##############################################################################################
+
+# # fig = plt.figure()
+# # ax = fig.add_subplot(111,projection='3d')
+# # ax.cla()
+# # x = session.mean_charuco_fr_mar_dim[:][:,0]
+# # y = session.mean_charuco_fr_mar_dim[:][:,1]
+# # z = session.mean_charuco_fr_mar_dim[:][:,2]
+# # ax.scatter(x,y,z, marker='o')
+# # plt.show()
+
+# # define Skeleton connections
+# # head = [17, 15, 0, 16, 18]
+# head = [ 15, 0, 16]
+# spine = [0,1,8]
+# rArm = [17, 15, 0, 16, 18]
+# rArm = [4 ,3 ,2 ,1]
+# lArm = [1, 5, 6, 7]
+# rLeg = [11 ,10, 9, 8]
+# lLeg = [14 ,13 ,12, 8]
+# rFoot = [11, 23,22, 11, 24]
+# lFoot = [14, 20, 19, 14, 21]
+
+# #Make some handy maps ;D
+# rHandIDstart = 25
+# lHandIDstart = rHandIDstart+21
+
+# thumb = np.array([0,1,2,3,4])
+# index = np.array([0, 5,6,7,8])
+# bird = np.array([0, 9,10,11,12])
+# ring = np.array([0, 13,14,15,16])
+# pinky = np.array([0, 17,18,19,20])
+
+
+# fig = plt.figure(figsize=(8,6))
+# axMain = fig.add_subplot(111,projection='3d')
+# plt.ion()
+# for fr in range(session.numFrames):
+# fig.suptitle(['Frame# - ', str(fr)])
+# axMain.cla()
+# char_x = session.mean_charuco_fr_mar_dim[:][:,0]
+# char_y = session.mean_charuco_fr_mar_dim[:][:,1]
+# char_z = session.mean_charuco_fr_mar_dim[:][:,2]
+
+# if session.useOpenPose:
+# sk_x = session.skel_fr_mar_dim[fr,:,0] #skeleton x data
+# sk_y = session.skel_fr_mar_dim[fr,:,1] #skeleton y data
+# sk_z = session.skel_fr_mar_dim[fr,:,2] #skeleton z data
+
+# axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)
+# axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)
+# axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)
+# axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)
+# axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)
+# axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)
+# axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)
+# axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)
+# axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)
+
+# # plot handybois
+# # right hand
+# axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+# axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+# axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+# axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+# axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+
+# #left hand
+# axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+# axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+# axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+# axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+# axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+
+# axMain.scatter(char_x,char_y,char_z, marker='o')
+
+# axMain.set_xbound(lower=-150, upper=150)
+# axMain.set_ybound(lower=-150, upper=150)
+# axMain.set_zbound(lower=-150, upper=150)
+
+# #imName = [session.sessionID + '_frame' + fr.zfill(6)]
+# frameName = str(fr).zfill(6)
+# saveFrameName = '{}.png'.format(frameName)
+# saveFramePath = session.imOutPath/saveFrameName
+# fig.savefig(saveFramePath)# , bbox_inches = 'tight')
+
+# plt.pause(0.01)
+# plt.show()
+
+
+def ReplaySkeleton(session, vidType, startframe,azimuth,elevation):
+ camImgPathList = {}
+ for cam in range(session.numCams):
+ camImgPathList[cam] = list(sorted(Path(session.openPose_imgPathList[cam]).glob('*.png')))
+ session.numFrames = len(camImgPathList[cam]) #will need to perhaps put a check in on whether numFrames between cameras are the same
+
+
+# define Skeleton connections
+# head = [17, 15, 0, 16, 18]
+ head = [ 15, 0, 16]
+ spine = [0,1,8]
+ rArm = [17, 15, 0, 16, 18]
+ rArm = [4 ,3 ,2 ,1]
+ lArm = [1, 5, 6, 7]
+ rLeg = [11 ,10, 9, 8]
+ lLeg = [14 ,13 ,12, 8]
+ rFoot = [11, 23,22, 11, 24]
+ lFoot = [14, 20, 19, 14, 21]
+
+ #Make some handy maps ;D
+ rHandIDstart = 25
+ lHandIDstart = rHandIDstart+21
+
+ thumb = np.array([0,1,2,3,4])
+ index = np.array([0, 5,6,7,8])
+ bird = np.array([0, 9,10,11,12])
+ ring = np.array([0, 13,14,15,16])
+ pinky = np.array([0, 17,18,19,20])
+
+
+ fig = plt.figure(figsize=(11,8))
+
+ if vidType == 0:
+ axMain = fig.add_subplot(111,projection='3d')
+ elif vidType == 1:
+ axMain = fig.add_subplot(121,projection='3d')
+ axCam1 = fig.add_subplot(122)
+
+ # fig.tight_layout(h_pad = 5)
+ axMain.view_init(azim = azimuth, elev= elevation)
+ plt.ion()
+ for fr in range(startframe,session.numFrames):
+ fig.suptitle(['Frame# - ', str(fr)])
+ axMain.cla()
+
+ if vidType == 1:
+ axCam1.cla()
+
+
+ char_x = session.mean_charuco_fr_mar_dim[:][:,0]
+ char_y = session.mean_charuco_fr_mar_dim[:][:,1]
+ char_z = session.mean_charuco_fr_mar_dim[:][:,2]
+
+ if session.useOpenPose:
+ sk_x = session.skel_fr_mar_dim[fr,:,0] #skeleton x data
+ sk_y = session.skel_fr_mar_dim[fr,:,1] #skeleton y data
+ sk_z = session.skel_fr_mar_dim[fr,:,2] #skeleton z data
+
+ axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)
+ axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)
+ axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)
+ axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)
+
+ # plot handybois
+ # right hand
+ axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+
+ #left hand
+ axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+
+ #plot charuco grid
+ axMain.scatter(char_x,char_y,char_z, marker='o')
+
+ # #plot camera positions (I think...)
+ # for camNum in range(len(session.cgroup.cameras)):
+ # axMain.scatter(session.cgroup.cameras[camNum].tvec[0], session.cgroup.cameras[camNum].tvec[1], session.cgroup.cameras[camNum].tvec[2], marker = 'p')
+
+ axMain.set_xlabel('x')
+ axMain.set_ylabel('y')
+ axMain.set_zlabel('z')
+
+ axMain.set_xlim(-100,100)
+ axMain.set_ylim(-100,100)
+ axMain.set_zlim(100,300)
+
+ if vidType ==0:
+ pass
+ elif vidType == 1:
+ axCam1.imshow(cv2.cvtColor(cv2.imread(str(camImgPathList[0][fr])),cv2.COLOR_BGR2RGB))
+ axCam1.axis('off')
+
+ plt.pause(0.01)
+ plt.show()
+
+ frameName = str(fr).zfill(6)
+ saveFrameName = '{}.png'.format(frameName)
+ saveFramePath = session.imOutPath/saveFrameName
+ fig.savefig(saveFramePath )
diff --git a/freemocap/reconstruct3D.py b/freemocap/reconstruct3D.py
new file mode 100644
index 00000000..a1a79142
--- /dev/null
+++ b/freemocap/reconstruct3D.py
@@ -0,0 +1,15 @@
+def reconstruct3D(session, data_nCams_nFrames_nImgPts_XY,data_params):
+ n_cams, n_frames, n_trackedPoints = data_params
+
+ #assert numDims == 2, 'Problem with your data! Not enough dimensions'
+
+ dataFlat_nCams_nTotalPoints_XY = data_nCams_nFrames_nImgPts_XY.reshape(n_cams, -1, 2) #reshape data to collapse across 'frames' so it becomes [numCams, numFrames*numPoints, XY]
+
+ data3d_flat = session.cgroup.triangulate(dataFlat_nCams_nTotalPoints_XY, progress=True)
+ dataReprojerr_flat = session.cgroup.reprojection_error(data3d_flat, dataFlat_nCams_nTotalPoints_XY, mean=True)
+
+ ##return:
+ data_fr_mar_dim = data3d_flat.reshape(n_frames, n_trackedPoints, 3)
+ dataReprojErr_fr_mar_err = dataReprojerr_flat.reshape(n_frames, n_trackedPoints)
+
+ return data_fr_mar_dim
diff --git a/freemocap/recordingconfig.py b/freemocap/recordingconfig.py
new file mode 100644
index 00000000..4473559d
--- /dev/null
+++ b/freemocap/recordingconfig.py
@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Mar 11 11:54:12 2021
+
+@author: Rontc
+"""
+from ruamel.yaml import YAML
+from pathlib import Path
+
+def createSession(session,filepath, existing_data = False):
+ #create file paths for the session, raw and synced videos
+
+ pathList = []
+
+ recordPath = filepath/'Data'
+ recordPath.mkdir(exist_ok= True)
+
+ session.sessionPath = recordPath/session.sessionID
+ #if existing_data:
+ #session.sessionPath.mkdir(exist_ok = True)
+ #else:
+ session.sessionPath.mkdir(exist_ok= True)
+ pathList.append(session.sessionPath)
+
+ session.rawVidPath = session.sessionPath/'RawVideos'
+ session.rawVidPath.mkdir(exist_ok = True)
+ pathList.append(session.rawVidPath)
+
+ session.syncedVidPath = session.sessionPath/'SyncedVideos'
+ session.syncedVidPath.mkdir(exist_ok = True)
+ pathList.append(session.syncedVidPath)
+
+ session.calVidPath = session.sessionPath/'CalVideos'
+ session.calVidPath.mkdir(exist_ok = True)
+ pathList.append(session.calVidPath)
+
+ session.openPoseDataPath = session.sessionPath/'OpenPoseData'
+ session.openPoseDataPath.mkdir(exist_ok = True)
+ pathList.append(session.openPoseDataPath)
+
+ session.dlcDataPath = session.sessionPath / 'DLCdata' / 'videos'
+ session.dlcDataPath.mkdir(exist_ok = True, parents = True)
+ pathList.append(session.dlcDataPath)
+
+ session.imOutPath = session.sessionPath / 'imOut'
+ session.imOutPath.mkdir(exist_ok=True)
+ pathList.append(session.imOutPath)
+
+ session.dataArrayPath = session.sessionPath/'Data Arrays'
+ session.dataArrayPath.mkdir(exist_ok=True)
+ pathList.append(session.dataArrayPath)
+
+ #create a list of paths that is passed to the recording script
+ #pathList = [session.sessionPath,session.rawVidPath,session.syncedVidPath,session.openPoseDataPath,session.dlcDataPath,session.imOutPath, session.dataArrayPath]
+
+
+ config_settings,config_yaml = create_config_yaml()
+ yaml_name = session.sessionPath/'{}_config.yaml'.format(session.sessionID)
+ session.yamlPath = yaml_name
+ # print(yaml_name)
+ write_config_yaml(session,config_settings,config_yaml,pathList,yaml_name)
+
+ return yaml_name
+
+def createSessionTxt(session,paramDict,rotDict):
+ #create a text file listing recording parameters
+ parameter_text = session.sessionPath/'sessionSettings.txt'
+ text = open(parameter_text, 'w')
+ text.write("Session ID = %s\n" %(session.sessionID))
+ text.write("%s = %s\n" %("Parameters", paramDict))
+ text.write("%s = %s\n" %("Rotations", rotDict))
+ text.close()
+
+
+
+
+def create_config_yaml():
+
+
+ yaml_str = """\
+ # Camera Inputs
+ CamInputs:
+ RotationInputs:
+ ParameterDict:
+
+ # Paths
+ Paths:
+ sessionPath:
+ rawVidPath:
+ syncedVidPath:
+ calVidPath:
+ openPoseDataPath:
+ dlcDataPath:
+ imOutPath:
+ dataArrayPath:
+ \n
+ """
+ config_yaml = YAML()
+ config_settings = config_yaml.load(yaml_str)
+ return config_settings,config_yaml
+
+
+def write_config_yaml(session,config_settings,config_yaml,pathList,yaml_name):
+
+ for count,key in enumerate(config_settings['Paths'].keys()):
+ config_settings['Paths'][key] = str(pathList[count])
+
+ config_settings['CamInputs']['RotationInputs'] = session.rotationInputs
+ config_settings['CamInputs']['ParameterDict'] = session.parameterDictionary
+
+
+ #yamlPath = sessionPath/'config_yaml'
+ #print(config_settings,yaml_name)
+ with open(yaml_name, 'w') as outfile:
+ config_yaml.dump(config_settings, outfile)
+
+
+def load_config_yaml(yaml_path):
+ config_yaml = YAML()
+ with open(yaml_path,'r') as fp:
+ config_settings = config_yaml.load(fp)
+ return config_settings
+
+def load_session_paths(session,config_settings):
+ configPaths = config_settings['Paths']
+ session.rawVidPath = Path(configPaths['rawVidPath'])
+ session.syncedVidPath = Path(configPaths['syncedVidPath'])
+ session.calVidPath = Path(configPaths['calVidPath'])
+ session.openPoseDataPath = Path(configPaths['openPoseDataPath'])
+ session.dlcDataPath = Path(configPaths['dlcDataPath'])
+ session.imOutPath = Path(configPaths['imOutPath'])
+ session.dataArrayPath = Path(configPaths['dataArrayPath'])
diff --git a/freemocap/runcams.py b/freemocap/runcams.py
new file mode 100644
index 00000000..96c2dee9
--- /dev/null
+++ b/freemocap/runcams.py
@@ -0,0 +1,81 @@
+
+from freemocap.webcam import startcamrecording, timesync, videotrim
+
+from pathlib import Path
+import time
+import pickle
+import pandas as pd
+import numpy as np
+from tkinter import Tk
+
+
+def RecordCams(session,camInputs,parameterDictionary,rotationInputs):
+
+ #%% Setting up recordings
+ beginTime = time.time()
+ numCams = len(camInputs) #number of cameras
+ numCamRange = range(numCams) #a range for the number of cameras that we have
+ vidNames = []
+ camIDs = []
+ for x in numCamRange: #create names for each of the initial untrimmed videos
+ singleCamID = 'Cam{}'.format(x+1)
+ camIDs.append(singleCamID) #creates IDs for each camera based on the number of cameras entered
+ singleVidName = 'raw_cam{}.mp4'.format(x+1)
+ vidNames.append(singleVidName)
+
+ #%% Starting the thread recordings for each camera
+ threads = []
+ for n in numCamRange: #starts recording video, opens threads for each camera
+ camRecordings = startcamrecording.CamRecordingThread(camIDs[n],camInputs[n],vidNames[n],session.rawVidPath,beginTime,parameterDictionary)
+ camRecordings.start()
+
+ threads.append(camRecordings)
+
+ for camRecordings in threads:
+ camRecordings.join() #make sure that one thread ending doesn't immediately end all the others (before they can dump data in a pickle file)
+
+ print('finished recordings')
+
+ timeStampList = []
+
+ for e in numCamRange: #open the saved pickle file for each camera, and add the timestamps to the dataList list
+ with open(camIDs[e], 'rb') as f:
+ camTimeList = pickle.load(f)
+ timeStampList.append(camTimeList)
+
+ timeDictionary = {}
+
+ id_and_time = zip(camIDs,timeStampList)
+
+ for cam,data in id_and_time:
+ timeDictionary[cam] = np.array(data) #create a dictionary that holds the timestamps for each camera
+
+
+ df = pd.DataFrame.from_dict(timeDictionary, orient = 'index') #create a data frame from this dictionary
+ timeStampData = df.transpose()
+ csvName = session.sessionID + '.csv' #create our csv filename
+ csvPath = session.sessionPath/csvName
+ timeStampData.to_csv(csvPath) #turn dataframe into a CSV
+
+ session.numCams = numCams
+ session.timeStampData = timeStampData
+ session.camIDs = camIDs
+ session.numCamRange = numCamRange
+ session.vidNames = vidNames
+
+def SyncCams(session, timeStampData,numCamRange,vidNames,camIDs):
+
+ #start the timesync process
+ frameTable,timeTable,frameRate,resultsTable,plots = timesync.TimeSync(timeStampData,numCamRange,camIDs)
+
+ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no
+ root = Tk()
+ proceed = timesync.proceedGUI(root,resultsTable,plots) #create a GUI instance called proceed
+ root.mainloop()
+
+ if proceed.proceed == True:
+ print()
+ print('Starting editing')
+ videotrim.VideoTrim(session,vidNames,frameTable,session.parameterDictionary,session.rotationInputs,numCamRange)
+ #videotrim.createCalibrationVideos(session,60,parameterDictionary)
+ print('all done')
\ No newline at end of file
diff --git a/freemocap/runopenpose.py b/freemocap/runopenpose.py
new file mode 100644
index 00000000..ab223ed3
--- /dev/null
+++ b/freemocap/runopenpose.py
@@ -0,0 +1,134 @@
+import os
+import subprocess
+import numpy as np
+from rich.progress import track
+from ruamel.yaml import YAML
+from pathlib import Path
+
+import glob
+import json
+
+def runOpenPose(session, dummyRun=False):
+
+ if not dummyRun:
+ os.chdir(session.openPoseExePath)
+
+ openPose_jsonPathList = [] #list to hold the paths to the json files
+ openPose_imgPathList = []
+ openPose_imgPathList_yaml = []
+ openPose_jsonPathList_yaml = []
+
+ for thisVidPath in session.syncedVidPath.iterdir(): #Run OpenPose ('Windows Portable Demo') on each video in the raw video folder
+ if thisVidPath.suffix =='.mp4': #NOTE - build some list of 'synced video names' and check against that
+ print('OpenPosing: ', thisVidPath.name )
+ vidPath = session.openPoseDataPath / thisVidPath.stem
+ jsonPath = vidPath / 'json'
+ jsonPath.mkdir(parents=True, exist_ok=True) #this camera's json files (with keypoints)
+ imgPath = vidPath / 'img'
+ imgPath.mkdir(parents=True, exist_ok=True)
+ openPose_jsonPathList.append(jsonPath)
+ openPose_imgPathList.append(imgPath)
+ openPose_imgPathList_yaml.append(str(imgPath))
+ openPose_jsonPathList_yaml.append(str(jsonPath))
+ if not dummyRun:
+ subprocess.run(['bin\OpenPoseDemo.exe', '--video', str(thisVidPath), '--write_json', str(jsonPath),'--write_images', str(imgPath),'--write_images_format','png','--net_resolution','-1x320','--hand','--face', '--number_people_max', '1'],shell = True)
+ else:
+ print('Skipping: ', thisVidPath.name )
+ session.openPose_jsonPathList = openPose_jsonPathList
+ session.openPose_imgPathList = openPose_imgPathList
+
+ yaml = YAML()
+ data = yaml.load(session.yamlPath)
+ data['openPose_imgPathList'] = openPose_imgPathList_yaml
+ data['openPose_jsonPathList'] = openPose_jsonPathList_yaml
+ yaml.dump(data, session.yamlPath)
+
+ os.chdir(session.sessionPath)
+
+
+def parseOpenPose(session):
+ thisCamNum = -1
+
+ ## %%
+ #build header for dataframe - NOTE - #openpose data comes in a line ordered 'pixel x location (px)', 'pixel y (py)', 'confidence (conf)' for each keypoint
+
+ dataFrameHeader = []
+ bodyCols = 75
+ handCols = 63 #per hand
+ faceCols = 210 #das alotta face!
+ headerLength = bodyCols + 2*handCols + faceCols#should be 411 for whatever version of openpose i was using on 11 Jan 2021
+ numImgPoints = headerLength
+
+ for bb in range(0,int(bodyCols/3)): #loop through the number of body markers (i.e. #bodyCols/3)
+ dataFrameHeader.append('body_' + str(bb).zfill(3) + '_pixx')
+ dataFrameHeader.append('body_' + str(bb).zfill(3) + '_pixy')
+ dataFrameHeader.append('body_' + str(bb).zfill(3) + '_conf')
+
+ for hr in range(0,int(handCols/3)): #loop through the number of handR markers (i.e. #handCols/3)
+ dataFrameHeader.append('handR_' + str(hr).zfill(3) + '_pixx')
+ dataFrameHeader.append('handR_' + str(hr).zfill(3) + '_pixy')
+ dataFrameHeader.append('handR_' + str(hr).zfill(3) + '_conf')
+
+ for hl in range(0,int(handCols/3)): #loop through the number of handL markers (i.e. #handCols/3)
+ dataFrameHeader.append('handL_' + str(hl).zfill(3) + '_pixx')
+ dataFrameHeader.append('handL_' + str(hl).zfill(3) + '_pixy')
+ dataFrameHeader.append('handL_' + str(hl).zfill(3) + '_conf')
+
+ for ff in range(0,int(faceCols/3)): #loop through the number of Face markers (i.e. #faceCols/3)
+ dataFrameHeader.append('face_' + str(ff).zfill(3) + '_pixx')
+ dataFrameHeader.append('face_' + str(ff).zfill(3) + '_pixy')
+ dataFrameHeader.append('face_' + str(ff).zfill(3) + '_conf')
+
+ assert len(dataFrameHeader) == headerLength, ['Header is the wrong length! Should be ' + str(headerLength) + ' but it is ' + str(len(dataFrameHeader)) + ' Check version of OpenPose?']
+
+ ## %%
+ ## load in data from json files
+ numFrames = int(len(list(Path(session.openPose_jsonPathList[0]).glob('*')))) #lol
+ numMarkers= int(int(len(dataFrameHeader)/3))
+ numCams = int(session.numCams)
+
+ openPoseData_nCams_nFrames_nImgPts_XYC = np.ndarray([numCams,numFrames,numMarkers,3]) #hardcoding for now because I am a bad person
+
+ for thisCams_JsonFolderPath in track(session.openPose_jsonPathList, description='Parsing json\'s into a dataframe (per cam)' ):
+ thisCamNum += 1
+ # print('Parsing into a dataframe: ', thisCams_JsonFolderPath.name )
+ jsonPaths = sorted(Path(thisCams_JsonFolderPath).glob('*.json')) #glob is a "generator(?)" for paths to all the jason for THIS camara
+
+ for thisJsonPath in jsonPaths: #loop throug all the json files and save their 'people' data to a dictionary (which will then be formatted into a pandas dataframe). NOTE - will be empty array if no hoomans visible in frame
+ # print('loading: ', thisJsonPath.name)
+ frameNum = int(thisJsonPath.stem.split('_')[-2]) #frame number we're on
+ thisJsonData = json.loads(thisJsonPath.read_bytes())
+ thisJsonData = thisJsonData['people'] # #FEATURE_REQUEST - at some point, we should check the openpose version (save it with the data somehow, verify everything is the same version, use different markernamlists for different versions, etc)
+
+ if thisJsonData: #if this json has data
+ bodyData = np.array(thisJsonData[0]['pose_keypoints_2d'])
+ handRData = np.array(thisJsonData[0]['hand_right_keypoints_2d'])
+ handLData = np.array(thisJsonData[0]['hand_left_keypoints_2d'])
+ faceData = np.array(thisJsonData[0]['face_keypoints_2d'])
+ thisFrameRow = np.hstack((bodyData,handRData, handLData, faceData)) #horizontally concatenate these arrays
+ else: #if this json is empty, just stuff it fulla NaNs
+ thisFrameRow = np.empty([headerLength])
+ thisFrameRow.fill(np.nan)
+
+
+ assert thisFrameRow.size == headerLength, ['Header is the wrong length! Should be ' + str(headerLength) + ' but it is ' + str(thisFrameRow.size) + ' Check version of OpenPose?']
+
+ if frameNum < openPoseData_nCams_nFrames_nImgPts_XYC.shape[1]: #NOTE: THIS SHOULDN"T BE NECESSARY. CURRENT Vid Trim Methods can (apparently) produce vids with off-by-one numbers of frames
+ openPoseData_nCams_nFrames_nImgPts_XYC[thisCamNum, frameNum, :, :] = np.reshape(thisFrameRow, [137,3]) #hard coding b/c I'm a bad person
+
+
+
+ openPoseData_nCams_nFrames_nImgPts_XY = openPoseData_nCams_nFrames_nImgPts_XYC[:,:,:,0:2].copy()
+ openpose_confidence = openPoseData_nCams_nFrames_nImgPts_XYC[:,:,:,2].copy()
+
+ openpose_score_threshold = .3
+ openPoseData_nCams_nFrames_nImgPts_XY[openpose_confidence < openpose_score_threshold] = np.nan #replace low confidence points with '
+
+ session.dataFrameHeader = dataFrameHeader
+ session.numImgPoints = numImgPoints
+ session.numFrames = frameNum #NOTE - Need to find a safer way to get this number
+
+ path_to_openpose_2d = session.dataArrayPath/'openPoseData_nCams_nFrames_nImgPts_XY.npy'
+ np.save(path_to_openpose_2d, openPoseData_nCams_nFrames_nImgPts_XY)
+
+ return openPoseData_nCams_nFrames_nImgPts_XY
\ No newline at end of file
diff --git a/freemocap/session.py b/freemocap/session.py
new file mode 100644
index 00000000..a0ab3501
--- /dev/null
+++ b/freemocap/session.py
@@ -0,0 +1,16 @@
+class Session: #session like "recording session"
+ def __init__(self, useOpenPose, useDLC):
+ self.sessionID = '' #The sessionID tag will be used to generate files names and whatnot
+ self.sessionPath = '' #The folder where the to-be-processed videos live (in a folder called "synced Vids")
+ self.numCams = ''#The number of cameras used in this recording session
+ self.openPoseDataPath = ''#Where the open pose data lives
+ self.dlcDataPath = ''#Where the DLC data lives
+ self.useOpenPose = useOpenPose
+ self.useDLC = useDLC
+ self.openPoseExePath = r'C:\openpose'
+
+ self.cam_inputs = None
+ self.parameterDictionary = None
+ self.rotationInputs = None
+
+
\ No newline at end of file
diff --git a/freemocap/stolenfromanipose.py b/freemocap/stolenfromanipose.py
new file mode 100644
index 00000000..d703bc40
--- /dev/null
+++ b/freemocap/stolenfromanipose.py
@@ -0,0 +1,1658 @@
+import cv2
+import numpy as np
+from copy import copy
+from scipy.sparse import lil_matrix, dok_matrix
+from scipy import optimize
+from scipy import signal
+from numba import jit
+from collections import defaultdict, Counter
+import toml
+import itertools
+from tqdm import trange
+from pprint import pprint
+import time
+
+from aniposelib.boards import merge_rows, extract_points, \
+ extract_rtvecs, get_video_params
+from aniposelib.utils import get_initial_extrinsics, make_M, get_rtvec, \
+ get_connections
+
+@jit(nopython=True, parallel=True)
+def triangulate_simple(points, camera_mats):
+ num_cams = len(camera_mats)
+ A = np.zeros((num_cams * 2, 4))
+ for i in range(num_cams):
+ x, y = points[i]
+ mat = camera_mats[i]
+ A[(i * 2):(i * 2 + 1)] = x * mat[2] - mat[0]
+ A[(i * 2 + 1):(i * 2 + 2)] = y * mat[2] - mat[1]
+ u, s, vh = np.linalg.svd(A, full_matrices=True)
+ p3d = vh[-1]
+ p3d = p3d[:3] / p3d[3]
+ return p3d
+
+
+def get_error_dict(errors_full, min_points=10):
+ n_cams = errors_full.shape[0]
+ errors_norm = np.linalg.norm(errors_full, axis=2)
+
+ good = ~np.isnan(errors_full[:, :, 0])
+
+ error_dict = dict()
+
+ for i in range(n_cams):
+ for j in range(i+1, n_cams):
+ subset = good[i] & good[j]
+ err_subset = errors_norm[:, subset][[i, j]]
+ err_subset_mean = np.mean(err_subset, axis=0)
+ if np.sum(subset) > min_points:
+ percents = np.percentile(err_subset_mean, [15, 75])
+ # percents = np.percentile(err_subset, [25, 75])
+ error_dict[(i, j)] = (err_subset.shape[1], percents)
+ return error_dict
+
+def check_errors(cgroup, imgp):
+ p3ds = cgroup.triangulate(imgp)
+ errors_full = cgroup.reprojection_error(p3ds, imgp, mean=False)
+ return get_error_dict(errors_full)
+
+def subset_extra(extra, ixs):
+ if extra is None:
+ return None
+
+ new_extra = {
+ 'objp': extra['objp'][ixs],
+ 'ids': extra['ids'][ixs],
+ 'rvecs': extra['rvecs'][:, ixs],
+ 'tvecs': extra['tvecs'][:, ixs]
+ }
+ return new_extra
+
+def resample_points_extra(imgp, extra, n_samp=25):
+ n_cams, n_points, _ = imgp.shape
+ ids = remap_ids(extra['ids'])
+ n_ids = np.max(ids)+1
+ good = ~np.isnan(imgp[:, :, 0])
+ ixs = np.arange(n_points)
+
+ cam_counts = np.zeros((n_ids, n_cams), dtype='int64')
+ for idnum in range(n_ids):
+ cam_counts[idnum] = np.sum(good[:, ids == idnum], axis=1)
+ cam_counts_random = cam_counts + np.random.random(size=cam_counts.shape)
+ best_boards = np.argsort(-cam_counts_random, axis=0)
+
+ cam_totals = np.zeros(n_cams, dtype='int64')
+
+ include = set()
+ for cam_num in range(n_cams):
+ for board_id in best_boards[:, cam_num]:
+ include.update(ixs[ids == board_id])
+ cam_totals += cam_counts[board_id]
+ if cam_totals[cam_num] >= n_samp or \
+ cam_counts_random[board_id, cam_num] < 1:
+ break
+
+ final_ixs = sorted(include)
+ newp = imgp[:, final_ixs]
+ extra = subset_extra(extra, final_ixs)
+ return newp, extra
+
+def resample_points(imgp, extra=None, n_samp=25):
+ # if extra is not None:
+ # return resample_points_extra(imgp, extra, n_samp)
+
+ n_cams = imgp.shape[0]
+ good = ~np.isnan(imgp[:, :, 0])
+ ixs = np.arange(imgp.shape[1])
+
+ num_cams = np.sum(~np.isnan(imgp[:, :, 0]), axis=0)
+
+ include = set()
+
+ for i in range(n_cams):
+ for j in range(i+1, n_cams):
+ subset = good[i] & good[j]
+ n_good = np.sum(subset)
+ if n_good > 0:
+ ## pick points, prioritizing points seen by more cameras
+ arr = np.copy(num_cams[subset]).astype('float64')
+ arr += np.random.random(size=arr.shape)
+ picked_ix = np.argsort(-arr)[:n_samp]
+ picked = ixs[subset][picked_ix]
+ include.update(picked)
+
+ final_ixs = sorted(include)
+ newp = imgp[:, final_ixs]
+ extra = subset_extra(extra, final_ixs)
+ return newp, extra
+
+def medfilt_data(values, size=15):
+ padsize = size+5
+ vpad = np.pad(values, (padsize, padsize), mode='reflect')
+ vpadf = signal.medfilt(vpad, kernel_size=size)
+ return vpadf[padsize:-padsize]
+
+def nan_helper(y):
+ return np.isnan(y), lambda z: z.nonzero()[0]
+
+def interpolate_data(vals):
+ nans, ix = nan_helper(vals)
+ out = np.copy(vals)
+ try:
+ out[nans] = np.interp(ix(nans), ix(~nans), vals[~nans])
+ except ValueError:
+ out[:] = 0
+ return out
+
+def remap_ids(ids):
+ unique_ids = np.unique(ids)
+ ids_out = np.copy(ids)
+ for i, num in enumerate(unique_ids):
+ ids_out[ids == num] = i
+ return ids_out
+
+def transform_points(points, rvecs, tvecs):
+ """Rotate points by given rotation vectors and translate.
+ Rodrigues' rotation formula is used.
+ """
+ theta = np.linalg.norm(rvecs, axis=1)[:, np.newaxis]
+ with np.errstate(invalid='ignore'):
+ v = rvecs / theta
+ v = np.nan_to_num(v)
+ dot = np.sum(points * v, axis=1)[:, np.newaxis]
+ cos_theta = np.cos(theta)
+ sin_theta = np.sin(theta)
+
+ rotated = cos_theta * points + \
+ sin_theta * np.cross(v, points) + \
+ dot * (1 - cos_theta) * v
+
+ return rotated + tvecs
+
+
+class Camera:
+ def __init__(self,
+ matrix=np.eye(3),
+ dist=np.zeros(5),
+ size=None,
+ rvec=np.zeros(3),
+ tvec=np.zeros(3),
+ name=None,
+ extra_dist=False):
+
+ self.set_camera_matrix(matrix)
+ self.set_distortions(dist)
+ self.set_size(size)
+ self.set_rotation(rvec)
+ self.set_translation(tvec)
+ self.set_name(name)
+ self.extra_dist = extra_dist
+
+ def get_dict(self):
+ return {
+ 'name': self.get_name(),
+ 'size': list(self.get_size()),
+ 'matrix': self.get_camera_matrix().tolist(),
+ 'distortions': self.get_distortions().tolist(),
+ 'rotation': self.get_rotation().tolist(),
+ 'translation': self.get_translation().tolist(),
+ }
+
+ def load_dict(self, d):
+ self.set_camera_matrix(d['matrix'])
+ self.set_rotation(d['rotation'])
+ self.set_translation(d['translation'])
+ self.set_distortions(d['distortions'])
+ self.set_name(d['name'])
+ self.set_size(d['size'])
+
+ def from_dict(d):
+ cam = Camera()
+ cam.load_dict(d)
+ return cam
+
+ def get_camera_matrix(self):
+ return self.matrix
+
+ def get_distortions(self):
+ return self.dist
+
+ def set_camera_matrix(self, matrix):
+ self.matrix = np.array(matrix, dtype='float64')
+
+ def set_focal_length(self, fx, fy=None):
+ if fy is None:
+ fy = fx
+ self.matrix[0, 0] = fx
+ self.matrix[1, 1] = fy
+
+ def get_focal_length(self, both=False):
+ fx = self.matrix[0, 0]
+ fy = self.matrix[1, 1]
+ if both:
+ return (fx, fy)
+ else:
+ return (fx + fy) / 2.0
+
+ def set_distortions(self, dist):
+ self.dist = np.array(dist, dtype='float64').ravel()
+
+ def set_rotation(self, rvec):
+ self.rvec = np.array(rvec, dtype='float64').ravel()
+
+ def get_rotation(self):
+ return self.rvec
+
+ def set_translation(self, tvec):
+ self.tvec = np.array(tvec, dtype='float64').ravel()
+
+ def get_translation(self):
+ return self.tvec
+
+ def get_extrinsics_mat(self):
+ return make_M(self.rvec, self.tvec)
+
+ def get_name(self):
+ return self.name
+
+ def set_name(self, name):
+ self.name = str(name)
+
+ def set_size(self, size):
+ """set size as (width, height)"""
+ self.size = size
+
+ def get_size(self):
+ """get size as (width, height)"""
+ return self.size
+
+ def resize_camera(self, scale):
+ """resize the camera by scale factor, updating intrinsics to match"""
+ size = self.get_size()
+ new_size = size[0] * scale, size[1] * scale
+ matrix = self.get_camera_matrix()
+ new_matrix = matrix * scale
+ new_matrix[2, 2] = 1
+ self.set_size(new_size)
+ self.set_camera_matrix(new_matrix)
+
+ def get_params(self):
+ params = np.zeros(8 + self.extra_dist, dtype='float64')
+ params[0:3] = self.get_rotation()
+ params[3:6] = self.get_translation()
+ params[6] = self.get_focal_length()
+ dist = self.get_distortions()
+ params[7] = dist[0]
+ if self.extra_dist:
+ params[8] = dist[1]
+ return params
+
+ def set_params(self, params):
+ self.set_rotation(params[0:3])
+ self.set_translation(params[3:6])
+ self.set_focal_length(params[6])
+
+ dist = np.zeros(5, dtype='float64')
+ dist[0] = params[7]
+ if self.extra_dist:
+ dist[1] = params[8]
+ self.set_distortions(dist)
+
+ def distort_points(self, points):
+ shape = points.shape
+ points = points.reshape(-1, 1, 2)
+ new_points = np.dstack([points, np.ones((points.shape[0], 1, 1))])
+ out, _ = cv2.projectPoints(new_points, np.zeros(3), np.zeros(3),
+ self.matrix.astype('float64'),
+ self.dist.astype('float64'))
+ return out.reshape(shape)
+
+ def undistort_points(self, points):
+ shape = points.shape
+ points = points.reshape(-1, 1, 2)
+ out = cv2.undistortPoints(points,
+ self.matrix.astype('float64'),
+ self.dist.astype('float64'))
+ return out.reshape(shape)
+
+ def project(self, points):
+ points = points.reshape(-1, 1, 3)
+ out, _ = cv2.projectPoints(points, self.rvec, self.tvec,
+ self.matrix.astype('float64'),
+ self.dist.astype('float64'))
+ return out
+
+ def reprojection_error(self, p3d, p2d):
+ proj = self.project(p3d).reshape(p2d.shape)
+ return p2d - proj
+
+ def copy(self):
+ return \
+ Camera(matrix=self.get_camera_matrix().copy(),
+ dist=self.get_distortions().copy(),
+ size=self.get_size(),
+ rvec=self.get_rotation().copy(),
+ tvec=self.get_translation().copy(),
+ name=self.get_name(),
+ extra_dist=self.extra_dist)
+
+class FisheyeCamera(Camera):
+ def __init__(self,
+ matrix=np.eye(3),
+ dist=np.zeros(4),
+ size=None,
+ rvec=np.zeros(3),
+ tvec=np.zeros(3),
+ name=None,
+ extra_dist=False):
+ self.set_camera_matrix(matrix)
+ self.set_distortions(dist)
+ self.set_size(size)
+ self.set_rotation(rvec)
+ self.set_translation(tvec)
+ self.set_name(name)
+ self.extra_dist = extra_dist
+
+ def from_dict(d):
+ cam = FisheyeCamera()
+ cam.load_dict(d)
+ return cam
+
+ def get_dict(self):
+ d = super().get_dict()
+ d['fisheye'] = True
+ return d
+
+ def distort_points(self, points):
+ shape = points.shape
+ points = points.reshape(-1, 1, 2)
+ new_points = np.dstack([points, np.ones((points.shape[0], 1, 1))])
+ out, _ = cv2.fisheye.projectPoints(new_points,
+ np.zeros(3), np.zeros(3),
+ self.matrix.astype('float64'),
+ self.dist.astype('float64'))
+ return out.reshape(shape)
+
+ def undistort_points(self, points):
+ shape = points.shape
+ points = points.reshape(-1, 1, 2)
+ out = cv2.fisheye.undistortPoints(points.astype('float64'),
+ self.matrix.astype('float64'),
+ self.dist.astype('float64'))
+ return out.reshape(shape)
+
+ def project(self, points):
+ points = points.reshape(-1, 1, 3)
+ out, _ = cv2.fisheye.projectPoints(points,
+ self.rvec, self.tvec,
+ self.matrix.astype('float64'),
+ self.dist.astype('float64'))
+ return out
+
+ def set_params(self, params):
+ self.set_rotation(params[0:3])
+ self.set_translation(params[3:6])
+ self.set_focal_length(params[6])
+
+ dist = np.zeros(4, dtype='float64')
+ dist[0] = params[7]
+ if self.extra_dist:
+ dist[1] = params[8]
+ # dist[2] = params[9]
+ # dist[3] = params[10]
+ self.set_distortions(dist)
+
+ def get_params(self):
+ params = np.zeros(8+self.extra_dist, dtype='float64')
+ params[0:3] = self.get_rotation()
+ params[3:6] = self.get_translation()
+ params[6] = self.get_focal_length()
+ dist = self.get_distortions()
+ params[7] = dist[0]
+ if self.extra_dist:
+ params[8] = dist[1]
+ # params[9] = dist[2]
+ # params[10] = dist[3]
+ return params
+
+ def copy(self):
+ return FisheyeCamera(
+ matrix=self.get_camera_matrix().copy(),
+ dist=self.get_distortions().copy(),
+ size=self.get_size(),
+ rvec=self.get_rotation().copy(),
+ tvec=self.get_translation().copy(),
+ name=self.get_name(),
+ extra_dist=self.extra_dist)
+
+class CameraGroup:
+ def __init__(self, cameras, metadata={}):
+ self.cameras = cameras
+ self.metadata = metadata
+
+ def subset_cameras(self, indices):
+ cams = [self.cameras[ix].copy() for ix in indices]
+ return CameraGroup(cams, self.metadata)
+
+ def subset_cameras_names(self, names):
+ cur_names = self.get_names()
+ cur_names_dict = dict(zip(cur_names, range(len(cur_names))))
+ indices = []
+ for name in names:
+ if name not in cur_names_dict:
+ raise IndexError(
+ "name {} not part of camera names: {}".format(
+ name, cur_names
+ ))
+ indices.append(cur_names_dict[name])
+ return self.subset_cameras(indices)
+
+ def project(self, points):
+ """Given an Nx3 array of points, this returns an CxNx2 array of 2D points,
+ where C is the number of cameras"""
+ points = points.reshape(-1, 1, 3)
+ n_points = points.shape[0]
+ n_cams = len(self.cameras)
+
+ out = np.empty((n_cams, n_points, 2), dtype='float64')
+ for cnum, cam in enumerate(self.cameras):
+ out[cnum] = cam.project(points).reshape(n_points, 2)
+
+ return out
+
+ def triangulate(self, points, undistort=True, progress=False):
+ """Given an CxNx2 array, this returns an Nx3 array of points,
+ where N is the number of points and C is the number of cameras"""
+
+ assert points.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), points.shape
+ )
+
+ one_point = False
+ if len(points.shape) == 2:
+ points = points.reshape(-1, 1, 2)
+ one_point = True
+
+ if undistort:
+ new_points = np.empty(points.shape)
+ for cnum, cam in enumerate(self.cameras):
+ # must copy in order to satisfy opencv underneath
+ sub = np.copy(points[cnum])
+ new_points[cnum] = cam.undistort_points(sub)
+ points = new_points
+
+ n_cams, n_points, _ = points.shape
+
+ out = np.empty((n_points, 3))
+ out[:] = np.nan
+
+ cam_mats = np.array([cam.get_extrinsics_mat() for cam in self.cameras])
+
+ if progress:
+ iterator = trange(n_points, ncols=70)
+ else:
+ iterator = range(n_points)
+
+ for ip in iterator:
+ subp = points[:, ip, :]
+ good = ~np.isnan(subp[:, 0])
+ if np.sum(good) >= 2:
+ out[ip] = triangulate_simple(subp[good], cam_mats[good])
+
+ if one_point:
+ out = out[0]
+
+ return out
+
+ def triangulate_possible(self, points, undistort=True,
+ min_cams=2, progress=False, threshold=0.5):
+ """Given an CxNxPx2 array, this returns an Nx3 array of points
+ by triangulating all possible points and picking the ones with
+ best reprojection error
+ where:
+ C: number of cameras
+ N: number of points
+ P: number of possible options per point
+ """
+
+ assert points.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), points.shape
+ )
+
+ n_cams, n_points, n_possible, _ = points.shape
+
+ cam_nums, point_nums, possible_nums = np.where(
+ ~np.isnan(points[:, :, :, 0]))
+
+ all_iters = defaultdict(dict)
+
+ for cam_num, point_num, possible_num in zip(cam_nums, point_nums,
+ possible_nums):
+ if cam_num not in all_iters[point_num]:
+ all_iters[point_num][cam_num] = []
+ all_iters[point_num][cam_num].append((cam_num, possible_num))
+
+ for point_num in all_iters.keys():
+ for cam_num in all_iters[point_num].keys():
+ all_iters[point_num][cam_num].append(None)
+
+ out = np.full((n_points, 3), np.nan, dtype='float64')
+ picked_vals = np.zeros((n_cams, n_points, n_possible), dtype='bool')
+ errors = np.zeros(n_points, dtype='float64')
+ points_2d = np.full((n_cams, n_points, 2), np.nan, dtype='float64')
+
+ if progress:
+ iterator = trange(n_points, ncols=70)
+ else:
+ iterator = range(n_points)
+
+ for point_ix in iterator:
+ best_point = None
+ best_error = 200
+
+ n_cams_max = len(all_iters[point_ix])
+
+ for picked in itertools.product(*all_iters[point_ix].values()):
+ picked = [p for p in picked if p is not None]
+ if len(picked) < min_cams and len(picked) != n_cams_max:
+ continue
+
+ cnums = [p[0] for p in picked]
+ xnums = [p[1] for p in picked]
+
+ pts = points[cnums, point_ix, xnums]
+ cc = self.subset_cameras(cnums)
+
+ p3d = cc.triangulate(pts, undistort=undistort)
+ err = cc.reprojection_error(p3d, pts, mean=True)
+
+ if err < best_error:
+ best_point = {
+ 'error': err,
+ 'point': p3d[:3],
+ 'points': pts,
+ 'picked': picked,
+ 'joint_ix': point_ix
+ }
+ best_error = err
+ if best_error < threshold:
+ break
+
+ if best_point is not None:
+ out[point_ix] = best_point['point']
+ picked = best_point['picked']
+ cnums = [p[0] for p in picked]
+ xnums = [p[1] for p in picked]
+ picked_vals[cnums, point_ix, xnums] = True
+ errors[point_ix] = best_point['error']
+ points_2d[cnums, point_ix] = best_point['points']
+
+ return out, picked_vals, points_2d, errors
+
+ def triangulate_ransac(self, points, undistort=True, min_cams=2, progress=False):
+ """Given an CxNx2 array, this returns an Nx3 array of points,
+ where N is the number of points and C is the number of cameras"""
+
+ assert points.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), points.shape
+ )
+
+ n_cams, n_points, _ = points.shape
+
+ points_ransac = points.reshape(n_cams, n_points, 1, 2)
+
+ return self.triangulate_possible(points_ransac,
+ undistort=undistort,
+ min_cams=min_cams,
+ progress=progress)
+
+
+ @jit(nopython=True, parallel=True, forceobj=True)
+ def reprojection_error(self, p3ds, p2ds, mean=False):
+ """Given an Nx3 array of 3D points and an CxNx2 array of 2D points,
+ where N is the number of points and C is the number of cameras,
+ this returns an CxNx2 array of errors.
+ Optionally mean=True, this averages the errors and returns array of length N of errors"""
+
+ one_point = False
+ if len(p3ds.shape) == 1 and len(p2ds.shape) == 2:
+ p3ds = p3ds.reshape(1, 3)
+ p2ds = p2ds.reshape(-1, 1, 2)
+ one_point = True
+
+ n_cams, n_points, _ = p2ds.shape
+ assert p3ds.shape == (n_points, 3), \
+ "shapes of 2D and 3D points are not consistent: " \
+ "2D={}, 3D={}".format(p2ds.shape, p3ds.shape)
+
+ errors = np.empty((n_cams, n_points, 2))
+
+ for cnum, cam in enumerate(self.cameras):
+ errors[cnum] = cam.reprojection_error(p3ds, p2ds[cnum])
+
+ if mean:
+ errors_norm = np.linalg.norm(errors, axis=2)
+ good = ~np.isnan(errors_norm)
+ errors_norm[~good] = 0
+ denom = np.sum(good, axis=0).astype('float64')
+ denom[denom < 1.5] = np.nan
+ errors = np.sum(errors_norm, axis=0) / denom
+
+ if one_point:
+ if mean:
+ errors = float(errors[0])
+ else:
+ errors = errors.reshape(-1, 2)
+
+ return errors
+
+
+ def bundle_adjust_iter(self, p2ds, extra=None,
+ n_iters=10, start_mu=15, end_mu=1,
+ max_nfev=200, ftol=1e-4,
+ n_samp_iter=100, n_samp_full=1000,
+ error_threshold=0.3,
+ verbose=False):
+ """Given an CxNx2 array of 2D points,
+ where N is the number of points and C is the number of cameras,
+ this performs iterative bundle adjustsment to fine-tune the parameters of the cameras.
+ That is, it performs bundle adjustment multiple times, adjusting the weights given to points
+ to reduce the influence of outliers.
+ This is inspired by the algorithm for Fast Global Registration by Zhou, Park, and Koltun
+ """
+
+ assert p2ds.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), p2ds.shape
+ )
+
+ p2ds_full = p2ds
+ extra_full = extra
+
+ p2ds, extra = resample_points(p2ds_full, extra_full,
+ n_samp=n_samp_full)
+ error = self.average_error(p2ds, median=True)
+
+ if verbose:
+ print('error: ', error)
+
+ mus = np.exp(np.linspace(np.log(start_mu), np.log(end_mu), num=n_iters))
+
+ if verbose:
+ print('n_samples: {}'.format(n_samp_iter))
+
+ for i in range(n_iters):
+ p2ds, extra = resample_points(p2ds_full, extra_full,
+ n_samp=n_samp_full)
+ p3ds = self.triangulate(p2ds)
+ errors_full = self.reprojection_error(p3ds, p2ds, mean=False)
+ errors_norm = self.reprojection_error(p3ds, p2ds, mean=True)
+
+ error_dict = get_error_dict(errors_full)
+ max_error = 0
+ min_error = 0
+ for k, v in error_dict.items():
+ num, percents = v
+ max_error = max(percents[-1], max_error)
+ min_error = max(percents[0], min_error)
+ mu = max(min(max_error, mus[i]), min_error)
+
+ good = errors_norm < mu
+ extra_good = subset_extra(extra, good)
+ p2ds_samp, extra_samp = resample_points(
+ p2ds[:, good], extra_good, n_samp=n_samp_iter)
+
+ error = np.median(errors_norm)
+
+ if error < error_threshold:
+ break
+
+ if verbose:
+ pprint(error_dict)
+ print('error: {:.2f}, mu: {:.1f}, ratio: {:.3f}'.format(error, mu, np.mean(good)))
+
+ self.bundle_adjust(p2ds_samp, extra_samp,
+ loss='linear', ftol=ftol,
+ max_nfev=max_nfev,
+ verbose=verbose)
+
+
+ p2ds, extra = resample_points(p2ds_full, extra_full,
+ n_samp=n_samp_full)
+ p3ds = self.triangulate(p2ds)
+ errors_full = self.reprojection_error(p3ds, p2ds, mean=False)
+ errors_norm = self.reprojection_error(p3ds, p2ds, mean=True)
+ error_dict = get_error_dict(errors_full)
+ if verbose:
+ pprint(error_dict)
+
+ max_error = 0
+ min_error = 0
+ for k, v in error_dict.items():
+ num, percents = v
+ max_error = max(percents[-1], max_error)
+ min_error = max(percents[0], min_error)
+ mu = max(max(max_error, end_mu), min_error)
+
+ good = errors_norm < mu
+ extra_good = subset_extra(extra, good)
+ self.bundle_adjust(p2ds[:, good], extra_good,
+ loss='linear',
+ ftol=ftol, max_nfev=max(200, max_nfev),
+ verbose=verbose)
+
+ error = self.average_error(p2ds, median=True)
+
+ p3ds = self.triangulate(p2ds)
+ errors_full = self.reprojection_error(p3ds, p2ds, mean=False)
+ error_dict = get_error_dict(errors_full)
+ if verbose:
+ pprint(error_dict)
+
+ if verbose:
+ print('error: ', error)
+
+ return error
+
+ def bundle_adjust(self, p2ds, extra=None,
+ loss='linear',
+ threshold=50,
+ ftol=1e-4,
+ max_nfev=1000,
+ weights=None,
+ start_params=None,
+ verbose=True):
+ """Given an CxNx2 array of 2D points,
+ where N is the number of points and C is the number of cameras,
+ this performs bundle adjustsment to fine-tune the parameters of the cameras"""
+
+ assert p2ds.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), p2ds.shape
+ )
+
+ if extra is not None:
+ extra['ids_map'] = remap_ids(extra['ids'])
+
+ x0, n_cam_params = self._initialize_params_bundle(p2ds, extra)
+
+ if start_params is not None:
+ x0 = start_params
+ n_cam_params = len(self.cameras[0].get_params())
+
+ error_fun = self._error_fun_bundle
+
+ jac_sparse = self._jac_sparsity_bundle(p2ds, n_cam_params, extra)
+
+ f_scale = threshold
+ opt = optimize.least_squares(error_fun,
+ x0,
+ jac_sparsity=jac_sparse,
+ f_scale=f_scale,
+ x_scale='jac',
+ loss=loss,
+ ftol=ftol,
+ method='trf',
+ tr_solver='lsmr',
+ verbose=2 * verbose,
+ max_nfev=max_nfev,
+ args=(p2ds, n_cam_params, extra))
+ best_params = opt.x
+
+ for i, cam in enumerate(self.cameras):
+ a = i * n_cam_params
+ b = (i + 1) * n_cam_params
+ cam.set_params(best_params[a:b])
+
+ error = self.average_error(p2ds)
+ return error
+
+ @jit(nopython=True, parallel=True, forceobj=True)
+ def _error_fun_bundle(self, params, p2ds, n_cam_params, extra):
+ """Error function for bundle adjustment"""
+ good = ~np.isnan(p2ds)
+ n_cams = len(self.cameras)
+
+ for i in range(n_cams):
+ cam = self.cameras[i]
+ a = i * n_cam_params
+ b = (i + 1) * n_cam_params
+ cam.set_params(params[a:b])
+
+ n_cams = len(self.cameras)
+ sub = n_cam_params * n_cams
+ n3d = p2ds.shape[1] * 3
+ p3ds_test = params[sub:sub+n3d].reshape(-1, 3)
+ errors = self.reprojection_error(p3ds_test, p2ds)
+ errors_reproj = errors[good]
+
+ if extra is not None:
+ ids = extra['ids_map']
+ objp = extra['objp']
+ min_scale = np.min(objp[objp > 0])
+ n_boards = int(np.max(ids)) + 1
+ a = sub+n3d
+ rvecs = params[a:a+n_boards*3].reshape(-1, 3)
+ tvecs = params[a+n_boards*3:a+n_boards*6].reshape(-1, 3)
+ expected = transform_points(objp, rvecs[ids], tvecs[ids])
+ errors_obj = 2 * (p3ds_test - expected).ravel() / min_scale
+ else:
+ errors_obj = np.array([])
+
+ return np.hstack([errors_reproj, errors_obj])
+
+
+ def _jac_sparsity_bundle(self, p2ds, n_cam_params, extra):
+ """Given an CxNx2 array of 2D points,
+ where N is the number of points and C is the number of cameras,
+ compute the sparsity structure of the jacobian for bundle adjustment"""
+
+ point_indices = np.zeros(p2ds.shape, dtype='int32')
+ cam_indices = np.zeros(p2ds.shape, dtype='int32')
+
+ for i in range(p2ds.shape[1]):
+ point_indices[:, i] = i
+
+ for j in range(p2ds.shape[0]):
+ cam_indices[j] = j
+
+ good = ~np.isnan(p2ds)
+
+ if extra is not None:
+ ids = extra['ids_map']
+ n_boards = int(np.max(ids)) + 1
+ total_board_params = n_boards * (3 + 3) # rvecs + tvecs
+ else:
+ n_boards = 0
+ total_board_params = 0
+
+ n_cams = p2ds.shape[0]
+ n_points = p2ds.shape[1]
+ total_params_reproj = n_cams * n_cam_params + n_points * 3
+ n_params = total_params_reproj + total_board_params
+
+ n_good_values = np.sum(good)
+ if extra is not None:
+ n_errors = n_good_values + n_points * 3
+ else:
+ n_errors = n_good_values
+
+ A_sparse = dok_matrix((n_errors, n_params), dtype='int16')
+
+ cam_indices_good = cam_indices[good]
+ point_indices_good = point_indices[good]
+
+ # -- reprojection error --
+ ix = np.arange(n_good_values)
+
+ ## update camera params based on point error
+ for i in range(n_cam_params):
+ A_sparse[ix, cam_indices_good * n_cam_params + i] = 1
+
+ ## update point position based on point error
+ for i in range(3):
+ A_sparse[ix, n_cams * n_cam_params + point_indices_good * 3 + i] = 1
+
+ # -- match for the object points--
+ if extra is not None:
+ point_ix = np.arange(n_points)
+
+ ## update all the camera parameters
+ # A_sparse[n_good_values:n_good_values+n_points*3,
+ # 0:n_cams*n_cam_params] = 1
+
+ ## update board rotation and translation based on error from expected
+ for i in range(3):
+ for j in range(3):
+ A_sparse[n_good_values + point_ix*3 + i,
+ total_params_reproj + ids*3 + j] = 1
+ A_sparse[n_good_values + point_ix*3 + i,
+ total_params_reproj + n_boards*3 + ids*3 + j] = 1
+
+
+ ## update point position based on error from expected
+ for i in range(3):
+ A_sparse[n_good_values + point_ix*3 + i,
+ n_cams*n_cam_params + point_ix*3 + i] = 1
+
+
+ return A_sparse
+
+ def _initialize_params_bundle(self, p2ds, extra):
+ """Given an CxNx2 array of 2D points,
+ where N is the number of points and C is the number of cameras,
+ initializes the parameters for bundle adjustment"""
+
+ cam_params = np.hstack([cam.get_params() for cam in self.cameras])
+ n_cam_params = len(cam_params) // len(self.cameras)
+
+ total_cam_params = len(cam_params)
+
+ n_cams, n_points, _ = p2ds.shape
+ assert n_cams == len(self.cameras), \
+ "number of cameras in CameraGroup does not " \
+ "match number of cameras in 2D points given"
+
+ p3ds = self.triangulate(p2ds)
+
+ if extra is not None:
+ ids = extra['ids_map']
+ n_boards = int(np.max(ids[~np.isnan(ids)])) + 1
+ total_board_params = n_boards * (3 + 3) # rvecs + tvecs
+
+ # initialize to 0
+ rvecs = np.zeros((n_boards, 3), dtype='float64')
+ tvecs = np.zeros((n_boards, 3), dtype='float64')
+
+ if 'rvecs' in extra and 'tvecs' in extra:
+ rvecs_all = extra['rvecs']
+ tvecs_all = extra['tvecs']
+ for board_num in range(n_boards):
+ point_id = np.where(ids == board_num)[0][0]
+ cam_ids_possible = np.where(~np.isnan(p2ds[:, point_id, 0]))[0]
+ cam_id = np.random.choice(cam_ids_possible)
+ M_cam = self.cameras[cam_id].get_extrinsics_mat()
+ M_board_cam = make_M(rvecs_all[cam_id, point_id],
+ tvecs_all[cam_id, point_id])
+ M_board = np.matmul(np.linalg.inv(M_cam), M_board_cam)
+ rvec, tvec = get_rtvec(M_board)
+ rvecs[board_num] = rvec
+ tvecs[board_num] = tvec
+
+
+ else:
+ total_board_params = 0
+
+ x0 = np.zeros(total_cam_params + p3ds.size + total_board_params)
+ x0[:total_cam_params] = cam_params
+ x0[total_cam_params:total_cam_params+p3ds.size] = p3ds.ravel()
+
+ if extra is not None:
+ start_board = total_cam_params+p3ds.size
+ x0[start_board:start_board + n_boards*3] = rvecs.ravel()
+ x0[start_board + n_boards*3:start_board + n_boards*6] = \
+ tvecs.ravel()
+
+ return x0, n_cam_params
+
+ def optim_points(self, points, p3ds,
+ constraints=[],
+ constraints_weak=[],
+ scale_smooth=4,
+ scale_length=2, scale_length_weak=0.5,
+ reproj_error_threshold=15, reproj_loss='soft_l1',
+ n_deriv_smooth=1, scores=None, verbose=False):
+ """
+ Take in an array of 2D points of shape CxNxJx2,
+ an array of 3D points of shape NxJx3,
+ and an array of constraints of shape Kx2, where
+ C: number of camera
+ N: number of frames
+ J: number of joints
+ K: number of constraints
+
+ This function creates an optimized array of 3D points of shape NxJx3.
+
+ Example constraints:
+ constraints = [[0, 1], [1, 2], [2, 3]]
+ (meaning that lengths of segments 0->1, 1->2, 2->3 are all constant)
+
+ """
+ assert points.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), points.shape
+ )
+
+ n_cams, n_frames, n_joints, _ = points.shape
+ constraints = np.array(constraints)
+ constraints_weak = np.array(constraints_weak)
+
+ p3ds_intp = np.apply_along_axis(interpolate_data, 0, p3ds)
+
+ p3ds_med = np.apply_along_axis(medfilt_data, 0, p3ds_intp, size=7)
+
+ default_smooth = 1.0/np.mean(np.abs(np.diff(p3ds_med, axis=0)))
+ scale_smooth_full = scale_smooth * default_smooth
+
+ t1 = time.time()
+
+ x0 = self._initialize_params_triangulation(
+ p3ds_intp, constraints, constraints_weak)
+
+ x0[~np.isfinite(x0)] = 0
+
+ jac = self._jac_sparsity_triangulation(
+ points, constraints, constraints_weak, n_deriv_smooth)
+
+ opt2 = optimize.least_squares(self._error_fun_triangulation,
+ x0=x0, jac_sparsity=jac,
+ loss='linear',
+ ftol=1e-3,
+ verbose=2*verbose,
+ args=(points,
+ constraints,
+ constraints_weak,
+ scores,
+ scale_smooth_full,
+ scale_length,
+ scale_length_weak,
+ reproj_error_threshold,
+ reproj_loss,
+ n_deriv_smooth))
+
+ p3ds_new2 = opt2.x[:p3ds.size].reshape(p3ds.shape)
+
+ t2 = time.time()
+
+ if verbose:
+ print('optimization took {:.2f} seconds'.format(t2 - t1))
+
+ return p3ds_new2
+
+
+ def optim_points_possible(self, points, p3ds,
+ constraints=[],
+ constraints_weak=[],
+ scale_smooth=4,
+ scale_length=2, scale_length_weak=0.5,
+ reproj_error_threshold=15, reproj_loss='soft_l1',
+ n_deriv_smooth=1, scores=None, verbose=False):
+ """
+ Take in an array of 2D points of shape CxNxJxPx2,
+ an array of 3D points of shape NxJx3,
+ and an array of constraints of shape Kx2, where
+ C: number of camera
+ N: number of frames
+ J: number of joints
+ P: number of possible options per point
+ K: number of constraints
+
+ This function creates an optimized array of 3D points of shape NxJx3.
+
+ Example constraints:
+ constraints = [[0, 1], [1, 2], [2, 3]]
+ (meaning that lengths of segments 0->1, 1->2, 2->3 are all constant)
+
+ """
+ assert points.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), points.shape
+ )
+
+ n_cams, n_frames, n_joints, n_possible, _ = points.shape
+ constraints = np.array(constraints)
+ constraints_weak = np.array(constraints_weak)
+
+ p3ds_intp = np.apply_along_axis(interpolate_data, 0, p3ds)
+
+ p3ds_med = np.apply_along_axis(medfilt_data, 0, p3ds_intp, size=7)
+
+ default_smooth = 1.0/np.mean(np.abs(np.diff(p3ds_med, axis=0)))
+ scale_smooth_full = scale_smooth * default_smooth
+
+ t1 = time.time()
+
+ x0 = self._initialize_params_triangulation_possible(
+ p3ds_intp, points, constraints=constraints, constraints_weak=constraints_weak)
+
+ print('getting jacobian...')
+ jac = self._jac_sparsity_triangulation_possible(
+ points,
+ constraints=constraints,
+ constraints_weak=constraints_weak,
+ n_deriv_smooth=n_deriv_smooth)
+
+ beta = 5
+
+ print('starting optimization...')
+ opt2 = optimize.least_squares(self._error_fun_triangulation_possible,
+ x0=x0, jac_sparsity=jac,
+ loss='linear',
+ ftol=1e-3,
+ verbose=2*verbose,
+ args=(points,
+ beta,
+ constraints,
+ constraints_weak,
+ scores,
+ scale_smooth_full,
+ scale_length,
+ scale_length_weak,
+ reproj_error_threshold,
+ reproj_loss,
+ n_deriv_smooth))
+ params = opt2.x
+
+ p3ds_new2 = params[:p3ds.size].reshape(p3ds.shape)
+
+ bad = np.isnan(points[:, :, :, :, 0])
+ all_bad = np.all(bad, axis=3)
+
+ n_params_norm = p3ds.size + len(constraints) + len(constraints_weak)
+
+ alphas = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='float64')
+ alphas[~bad] = params[n_params_norm:]
+
+ alphas_exp = np.exp(beta * alphas)
+ alphas_exp[bad] = 0
+ alphas_sum = np.sum(alphas_exp, axis=3)
+ alphas_sum[all_bad] = 1
+ alphas_norm = alphas_exp / alphas_sum[:, :, :, None]
+ alphas_norm[bad] = np.nan
+
+ t2 = time.time()
+
+ if verbose:
+ print('optimization took {:.2f} seconds'.format(t2 - t1))
+
+ return p3ds_new2, alphas_norm
+
+
+ def triangulate_optim(self, points, init_ransac=False, init_progress=False,
+ **kwargs):
+ """
+ Take in an array of 2D points of shape CxNxJx2, and an array of constraints of shape Kx2, where
+ C: number of camera
+ N: number of frames
+ J: number of joints
+ K: number of constraints
+
+ This function creates an optimized array of 3D points of shape NxJx3.
+
+ Example constraints:
+ constraints = [[0, 1], [1, 2], [2, 3]]
+ (meaning that lengths of segments 0->1, 1->2, 2->3 are all constant)
+
+ """
+
+ assert points.shape[0] == len(self.cameras), \
+ "Invalid points shape, first dim should be equal to" \
+ " number of cameras ({}), but shape is {}".format(
+ len(self.cameras), points.shape
+ )
+
+ n_cams, n_frames, n_joints, _ = points.shape
+ # constraints = np.array(constraints)
+ # constraints_weak = np.array(constraints_weak)
+
+ points_shaped = points.reshape(n_cams, n_frames*n_joints, 2)
+ if init_ransac:
+ p3ds, picked, p2ds, errors = self.triangulate_ransac(points_shaped, progress=init_progress)
+ points = p2ds.reshape(points.shape)
+ else:
+ p3ds = self.triangulate(points_shaped, progress=init_progress)
+ p3ds = p3ds.reshape((n_frames, n_joints, 3))
+
+ c = np.isfinite(p3ds[:, :, 0])
+ if np.sum(c) < 20:
+ print("warning: not enough 3D points to run optimization")
+ return p3ds
+
+ return self.optim_points(points, p3ds, **kwargs)
+
+
+
+ @jit(nopython=True, forceobj=True, parallel=True)
+ def _error_fun_triangulation(self, params, p2ds,
+ constraints=[],
+ constraints_weak=[],
+ scores=None,
+ scale_smooth=10000,
+ scale_length=1,
+ scale_length_weak=0.2,
+ reproj_error_threshold=100,
+ reproj_loss='soft_l1',
+ n_deriv_smooth=1):
+ n_cams, n_frames, n_joints, _ = p2ds.shape
+
+ n_3d = n_frames*n_joints*3
+ n_constraints = len(constraints)
+ n_constraints_weak = len(constraints_weak)
+
+ # load params
+ p3ds = params[:n_3d].reshape((n_frames, n_joints, 3))
+ joint_lengths = np.array(params[n_3d:n_3d+n_constraints])
+ joint_lengths_weak = np.array(params[n_3d+n_constraints:])
+
+ # reprojection errors
+ p3ds_flat = p3ds.reshape(-1, 3)
+ p2ds_flat = p2ds.reshape((n_cams, -1, 2))
+ errors = self.reprojection_error(p3ds_flat, p2ds_flat)
+ if scores is not None:
+ scores_flat = scores.reshape((n_cams, -1))
+ errors = errors * scores_flat[:, :, None]
+ errors_reproj = errors[~np.isnan(p2ds_flat)]
+
+ rp = reproj_error_threshold
+ errors_reproj = np.abs(errors_reproj)
+ if reproj_loss == 'huber':
+ bad = errors_reproj > rp
+ errors_reproj[bad] = rp*(2*np.sqrt(errors_reproj[bad]/rp) - 1)
+ elif reproj_loss == 'linear':
+ pass
+ elif reproj_loss == 'soft_l1':
+ errors_reproj = rp*2*(np.sqrt(1+errors_reproj/rp)-1)
+
+ # temporal constraint
+ errors_smooth = np.diff(p3ds, n=n_deriv_smooth, axis=0).ravel() * scale_smooth
+
+ # joint length constraint
+ errors_lengths = np.empty((n_constraints, n_frames), dtype='float64')
+ for cix, (a, b) in enumerate(constraints):
+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)
+ expected = joint_lengths[cix]
+ errors_lengths[cix] = 100*(lengths - expected)/expected
+ errors_lengths = errors_lengths.ravel() * scale_length
+
+ errors_lengths_weak = np.empty((n_constraints_weak, n_frames), dtype='float64')
+ for cix, (a, b) in enumerate(constraints_weak):
+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)
+ expected = joint_lengths_weak[cix]
+ errors_lengths_weak[cix] = 100*(lengths - expected)/expected
+ errors_lengths_weak = errors_lengths_weak.ravel() * scale_length_weak
+
+ return np.hstack([errors_reproj, errors_smooth,
+ errors_lengths, errors_lengths_weak])
+
+ def _error_fun_triangulation_possible(self, params, p2ds,
+ beta=2,
+ constraints=[],
+ constraints_weak=[],
+ *args):
+ # extract alphas from end of params
+ # soft argmax for picking the appropriate points from p2ds
+ # pass the points to error_fun_triangulate_possible for residuals
+ # add errors to keep the alphas in check
+ # return all the errors
+
+ n_cams, n_frames, n_joints, n_possible, _ = p2ds.shape
+
+ n_3d = n_frames*n_joints*3
+ n_constraints = len(constraints)
+ n_constraints_weak = len(constraints_weak)
+ n_params_norm = n_3d+n_constraints+n_constraints_weak
+
+ # load params
+ bad = np.isnan(p2ds[:, :, :, :, 0])
+ all_bad = np.all(bad, axis=3)
+
+ alphas = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='float64')
+ alphas[~bad] = params[n_params_norm:]
+ params_rest = np.array(params[:n_params_norm])
+
+ # get normalized alphas
+ alphas_exp = np.exp(beta * alphas)
+ alphas_exp[bad] = 0
+ alphas_sum = np.sum(alphas_exp, axis=3)
+ alphas_sum[all_bad] = 1
+ alphas_norm = alphas_exp / alphas_sum[:, :, :, None]
+
+ # extract the 2D points using soft argmax
+ p2ds_test = np.copy(p2ds)
+ p2ds_test[bad] = 0
+ p2ds_adj = np.sum(alphas_norm[:, :, :, :, None] * p2ds_test, axis=3)
+ p2ds_adj[all_bad] = np.nan
+
+ errors = self._error_fun_triangulation(params_rest, p2ds_adj,
+ constraints, constraints_weak, *args)
+
+ alphas_test = alphas_norm[~all_bad]
+ errors_alphas = (1 - np.std(alphas_test, axis=1)) * 10
+
+ return np.hstack([errors, errors_alphas])
+
+
+ def _initialize_params_triangulation(self, p3ds,
+ constraints=[],
+ constraints_weak=[]):
+ joint_lengths = np.empty(len(constraints), dtype='float64')
+ joint_lengths_weak = np.empty(len(constraints_weak), dtype='float64')
+
+ for cix, (a, b) in enumerate(constraints):
+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)
+ joint_lengths[cix] = np.median(lengths)
+
+
+ for cix, (a, b) in enumerate(constraints_weak):
+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)
+ joint_lengths_weak[cix] = np.median(lengths)
+
+ all_lengths = np.hstack([joint_lengths, joint_lengths_weak])
+ med = np.median(all_lengths)
+ if med == 0:
+ med = 1e-3
+
+ mad = np.median(np.abs(all_lengths - med))
+
+ joint_lengths[joint_lengths == 0] = med
+ joint_lengths_weak[joint_lengths_weak == 0] = med
+ joint_lengths[joint_lengths > med+mad*5] = med
+ joint_lengths_weak[joint_lengths_weak > med+mad*5] = med
+
+ return np.hstack([p3ds.ravel(), joint_lengths, joint_lengths_weak])
+
+ def _initialize_params_triangulation_possible(self, p3ds, p2ds, **kwargs):
+ # initialize params using above function
+ # initialize alphas to 1 for first one and 0 for other possible
+
+ n_cams, n_frames, n_joints, n_possible, _ = p2ds.shape
+ good = ~np.isnan(p2ds[:, :, :, :, 0])
+
+ alphas = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='float64')
+ alphas[:, :, :, 0] = 0
+
+ params = self._initialize_params_triangulation(p3ds, **kwargs)
+ params_full = np.hstack([params, alphas[good]])
+
+ return params_full
+
+ def _jac_sparsity_triangulation(self, p2ds,
+ constraints=[],
+ constraints_weak=[],
+ n_deriv_smooth=1):
+ n_cams, n_frames, n_joints, _ = p2ds.shape
+ n_constraints = len(constraints)
+ n_constraints_weak = len(constraints_weak)
+
+ p2ds_flat = p2ds.reshape((n_cams, -1, 2))
+
+ point_indices = np.zeros(p2ds_flat.shape, dtype='int32')
+ for i in range(p2ds_flat.shape[1]):
+ point_indices[:, i] = i
+
+ point_indices_3d = np.arange(n_frames*n_joints)\
+ .reshape((n_frames, n_joints))
+
+ good = ~np.isnan(p2ds_flat)
+ n_errors_reproj = np.sum(good)
+ n_errors_smooth = (n_frames-n_deriv_smooth) * n_joints * 3
+ n_errors_lengths = n_constraints * n_frames
+ n_errors_lengths_weak = n_constraints_weak * n_frames
+
+ n_errors = n_errors_reproj + n_errors_smooth + \
+ n_errors_lengths + n_errors_lengths_weak
+
+ n_3d = n_frames*n_joints*3
+ n_params = n_3d + n_constraints + n_constraints_weak
+
+ point_indices_good = point_indices[good]
+
+ A_sparse = dok_matrix((n_errors, n_params), dtype='int16')
+
+ # constraints for reprojection errors
+ ix_reproj = np.arange(n_errors_reproj)
+ for k in range(3):
+ A_sparse[ix_reproj, point_indices_good * 3 + k] = 1
+
+ # sparse constraints for smoothness in time
+ frames = np.arange(n_frames-n_deriv_smooth)
+ for j in range(n_joints):
+ for n in range(n_deriv_smooth+1):
+ pa = point_indices_3d[frames, j]
+ pb = point_indices_3d[frames+n, j]
+ for k in range(3):
+ A_sparse[n_errors_reproj + pa*3 + k, pb*3 + k] = 1
+
+ ## -- strong constraints --
+ # joint lengths should change with joint lengths errors
+ start = n_errors_reproj + n_errors_smooth
+ frames = np.arange(n_frames)
+ for cix, (a, b) in enumerate(constraints):
+ A_sparse[start + cix*n_frames + frames, n_3d+cix] = 1
+
+ # points should change accordingly to match joint lengths too
+ frames = np.arange(n_frames)
+ for cix, (a, b) in enumerate(constraints):
+ pa = point_indices_3d[frames, a]
+ pb = point_indices_3d[frames, b]
+ for k in range(3):
+ A_sparse[start + cix*n_frames + frames, pa*3 + k] = 1
+ A_sparse[start + cix*n_frames + frames, pb*3 + k] = 1
+
+ ## -- weak constraints --
+ # joint lengths should change with joint lengths errors
+ start = n_errors_reproj + n_errors_smooth + n_errors_lengths
+ frames = np.arange(n_frames)
+ for cix, (a, b) in enumerate(constraints_weak):
+ A_sparse[start + cix*n_frames + frames, n_3d + n_constraints + cix] = 1
+
+ # points should change accordingly to match joint lengths too
+ frames = np.arange(n_frames)
+ for cix, (a, b) in enumerate(constraints_weak):
+ pa = point_indices_3d[frames, a]
+ pb = point_indices_3d[frames, b]
+ for k in range(3):
+ A_sparse[start + cix*n_frames + frames, pa*3 + k] = 1
+ A_sparse[start + cix*n_frames + frames, pb*3 + k] = 1
+
+ return A_sparse
+
+ def _jac_sparsity_triangulation_possible(self, p2ds_full, **kwargs):
+ # initialize sparse jacobian using above function
+ # extend to include alphas from parameters
+ ## TODO: this initialization is really slow for some reason
+
+ n_cams, n_frames, n_joints, n_possible, _ = p2ds_full.shape
+ good_full = ~np.isnan(p2ds_full[:, :, :, :, 0])
+ any_good = np.any(good_full, axis=3)
+
+ n_alphas = np.sum(good_full)
+ n_errors_alphas = np.sum(any_good)
+
+ p2ds = p2ds_full[:, :, :, 0]
+ A_sparse = self._jac_sparsity_triangulation(p2ds, **kwargs)
+
+ n_errors, n_params = A_sparse.shape
+
+ B_sparse = dok_matrix((n_errors + n_errors_alphas, n_params + n_alphas), dtype='int16')
+ for r, c in zip(*A_sparse.nonzero()):
+ B_sparse[r, c] = A_sparse[r, c]
+
+ point_indices_2d = np.arange(n_cams*n_frames*n_joints)\
+ .reshape(n_cams, n_frames, n_joints)
+ point_indices_2d_rep = np.repeat(point_indices_2d[:, :, :, None], 2, axis=3)
+ point_indices_2d_good = point_indices_2d_rep[~np.isnan(p2ds)]
+ point_indices_good = point_indices_2d[any_good]
+
+ alpha_indices = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='int64')
+ for pnum in range(n_possible):
+ alpha_indices[:, :, :, pnum] = point_indices_2d
+
+ alpha_indices_good = alpha_indices[good_full]
+
+ # alphas should change according to the reprojection error for each corresponding point
+ point_indices_2d_good_find = defaultdict(list)
+ for ix, p in enumerate(point_indices_2d_good):
+ point_indices_2d_good_find[p].append(ix)
+
+ for ix, alpha_index in enumerate(alpha_indices_good):
+ B_sparse[point_indices_2d_good_find[alpha_index],
+ n_params + ix] = 1
+
+ # alphas should change according to the alpha errors
+ point_indices_good_find = dict()
+ for ix, p in enumerate(point_indices_good):
+ point_indices_good_find[p] = ix
+
+ for ix, alpha_index in enumerate(alpha_indices_good):
+ if alpha_index in point_indices_good_find:
+ err_ix = n_errors + point_indices_good_find[alpha_index]
+ B_sparse[err_ix, n_params + ix] = 1
+
+ return B_sparse
+
+ def copy(self):
+ cameras = [cam.copy() for cam in self.cameras]
+ metadata = copy(self.metadata)
+ return CameraGroup(cameras, metadata)
+
+ def set_rotations(self, rvecs):
+ for cam, rvec in zip(self.cameras, rvecs):
+ cam.set_rotation(rvec)
+
+ def set_translations(self, tvecs):
+ for cam, tvec in zip(self.cameras, tvecs):
+ cam.set_translation(tvec)
+
+ def get_rotations(self):
+ rvecs = []
+ for cam in self.cameras:
+ rvec = cam.get_rotation()
+ rvecs.append(rvec)
+ return np.array(rvecs)
+
+ def get_translations(self):
+ tvecs = []
+ for cam in self.cameras:
+ tvec = cam.get_translation()
+ tvecs.append(tvec)
+ return np.array(tvecs)
+
+ def get_names(self):
+ return [cam.get_name() for cam in self.cameras]
+
+ def set_names(self, names):
+ for cam, name in zip(self.cameras, names):
+ cam.set_name(name)
+
+ def average_error(self, p2ds, median=False):
+ p3ds = self.triangulate(p2ds)
+ errors = self.reprojection_error(p3ds, p2ds, mean=True)
+ if median:
+ return np.median(errors)
+ else:
+ return np.mean(errors)
+
+ def calibrate_rows(self, all_rows, board,
+ init_intrinsics=True, init_extrinsics=True, verbose=True,
+ **kwargs):
+ """Assumes camera sizes are set properly"""
+ for rows, camera in zip(all_rows, self.cameras):
+ size = camera.get_size()
+
+ assert size is not None, \
+ "Camera with name {} has no specified frame size".format(camera.get_name())
+
+ if init_intrinsics:
+ objp, imgp = board.get_all_calibration_points(rows)
+ mixed = [(o, i) for (o, i) in zip(objp, imgp) if len(o) > 5]
+ objp, imgp = zip(*mixed)
+ matrix = cv2.initCameraMatrix2D(objp, imgp, tuple(size))
+ camera.set_camera_matrix(matrix)
+
+ for i, (row, cam) in enumerate(zip(all_rows, self.cameras)):
+ all_rows[i] = board.estimate_pose_rows(cam, row)
+
+ merged = merge_rows(all_rows)
+ imgp, extra = extract_points(merged, board, min_cameras=2)
+
+ if init_extrinsics:
+ rtvecs = extract_rtvecs(merged)
+ if verbose:
+ pprint(get_connections(rtvecs, self.get_names()))
+ rvecs, tvecs = get_initial_extrinsics(rtvecs)
+ self.set_rotations(rvecs)
+ self.set_translations(tvecs)
+
+ error = self.bundle_adjust_iter(imgp, extra, verbose=verbose, **kwargs)
+
+ return error,merged
+
+ def get_rows_videos(self, videos, board, verbose=True):
+ all_rows = []
+
+ for cix, (cam, cam_videos) in enumerate(zip(self.cameras, videos)):
+ rows_cam = []
+ for vnum, vidname in enumerate(cam_videos):
+ if verbose: print(vidname)
+ rows = board.detect_video(vidname, prefix=vnum, progress=verbose)
+ if verbose: print("{} boards detected".format(len(rows)))
+ rows_cam.extend(rows)
+ all_rows.append(rows_cam)
+
+ return all_rows
+
+ def set_camera_sizes_videos(self, videos):
+ for cix, (cam, cam_videos) in enumerate(zip(self.cameras, videos)):
+ rows_cam = []
+ for vnum, vidname in enumerate(cam_videos):
+ params = get_video_params(vidname)
+ size = (params['width'], params['height'])
+ cam.set_size(size)
+
+ def calibrate_videos(self, videos, board,
+ init_intrinsics=True, init_extrinsics=True, verbose=True,
+ **kwargs):
+ """Takes as input a list of list of video filenames, one list of each camera.
+ Also takes a board which specifies what should be detected in the videos"""
+
+ all_rows = self.get_rows_videos(videos, board, verbose=verbose)
+ if init_extrinsics:
+ self.set_camera_sizes_videos(videos)
+
+ error,merged = self.calibrate_rows(all_rows, board,
+ init_intrinsics=init_intrinsics,
+ init_extrinsics=init_extrinsics,
+ verbose=verbose, **kwargs)
+ return error, merged
+
+ def get_dicts(self):
+ out = []
+ for cam in self.cameras:
+ out.append(cam.get_dict())
+ return out
+
+ def from_dicts(arr):
+ cameras = []
+ for d in arr:
+ if 'fisheye' in d and d['fisheye']:
+ cam = FisheyeCamera.from_dict(d)
+ else:
+ cam = Camera.from_dict(d)
+ cameras.append(cam)
+ return CameraGroup(cameras)
+
+ def from_names(names, fisheye=False):
+ cameras = []
+ for name in names:
+ if fisheye:
+ cam = FisheyeCamera(name=name)
+ else:
+ cam = Camera(name=name)
+ cameras.append(cam)
+ return CameraGroup(cameras)
+
+ def load_dicts(self, arr):
+ for cam, d in zip(self.cameras, arr):
+ cam.load_dict(d)
+
+ def dump(self, fname):
+ dicts = self.get_dicts()
+ names = ['cam_{}'.format(i) for i in range(len(dicts))]
+ master_dict = dict(zip(names, dicts))
+ master_dict['metadata'] = self.metadata
+ with open(fname, 'w') as f:
+ toml.dump(master_dict, f)
+
+ def load(fname):
+ master_dict = toml.load(fname)
+ keys = sorted(master_dict.keys())
+ items = [master_dict[k] for k in keys if k != 'metadata']
+ cgroup = CameraGroup.from_dicts(items)
+ if 'metadata' in master_dict:
+ cgroup.metadata = master_dict['metadata']
+ return cgroup
+
+ def resize_cameras(self, scale):
+ for cam in self.cameras:
+ cam.resize_camera(scale)
diff --git a/freemocap/synccams.py b/freemocap/synccams.py
new file mode 100644
index 00000000..56b8f931
--- /dev/null
+++ b/freemocap/synccams.py
@@ -0,0 +1,11 @@
+from freemocap.webcam import timesync
+from tkinter import Tk
+
+
+def RunSync(session, timeStampData,numCamRange,camIDs):
+ frameTable,timeTable,frameRate,resultsTable,plots = timesync.TimeSync(timeStampData,numCamRange,camIDs) #start the timesync process
+ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no
+ root = Tk()
+ proceed = timesync.proceedGUI(root,resultsTable,plots) #create a GUI instance called proceed
+ root.mainloop()
+
\ No newline at end of file
diff --git a/freemocap/webcam/__init__.py b/freemocap/webcam/__init__.py
new file mode 100644
index 00000000..ef4b4f30
--- /dev/null
+++ b/freemocap/webcam/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Mar 1 13:39:34 2021
+
+@author: Rontc
+"""
+
diff --git a/freemocap/webcam/camsetup.py b/freemocap/webcam/camsetup.py
new file mode 100644
index 00000000..df333ec7
--- /dev/null
+++ b/freemocap/webcam/camsetup.py
@@ -0,0 +1,56 @@
+
+
+import threading
+import cv2
+import imutils
+
+class VideoSetup(threading.Thread):
+ def __init__(self, camID,parameterDictionary,rotNum):
+ self.camID = camID
+ self.parameterDictionary = parameterDictionary
+ self.rotNum = rotNum
+ threading.Thread.__init__(self)
+ def run(self):
+ #print("Starting " + self.previewName)
+ self.record(self.parameterDictionary,self.rotNum)
+ def record(self,parameterDictionary,rotNum):
+ exposure = parameterDictionary.get('exposure')
+ resWidth = parameterDictionary.get('resWidth')
+ resHeight = parameterDictionary.get('resHeight')
+ camName = "Camera" + str(self.camID)
+
+ cv2.namedWindow(camName)
+ cap = cv2.VideoCapture(self.camID,cv2.CAP_DSHOW)
+ cap.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)
+ cap.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)
+ cap.set(cv2.CAP_PROP_EXPOSURE, exposure)
+ #print('exposure',cap.get(cv2.CAP_PROP_EXPOSURE))
+ #if not cap.isOpened():
+ # raise RuntimeError('No camera found at input '+ str(self.camID))
+ while True:
+ ret1,frame1 = cap.read()
+ if ret1 ==True:
+ if rotNum is not None:
+ frame1 = imutils.rotate_bound(frame1, angle= rotNum)
+ cv2.imshow(camName,frame1)
+ if cv2.waitKey(1) & 0xFF== 27:
+ # == ord('q') for q
+ break
+
+ else:
+ break
+ cv2.destroyWindow(camName)
+
+def RunSetup(cam_inputs,rotation_input,paramDict):
+ if not cam_inputs:
+ raise ValueError('Camera input list (cam_inputs) is empty')
+
+ ulist = []
+
+ for cam_input,cam_rotation in zip(cam_inputs,rotation_input):
+ u = VideoSetup(cam_input,paramDict,cam_rotation)
+ u.start()
+ ulist.append(u)
+
+ for k in ulist:
+ k.join()
\ No newline at end of file
diff --git a/freemocap/webcam/checkcams.py b/freemocap/webcam/checkcams.py
new file mode 100644
index 00000000..ed466a06
--- /dev/null
+++ b/freemocap/webcam/checkcams.py
@@ -0,0 +1,27 @@
+from tqdm import tqdm
+import cv2
+
+def TestDevice(source):
+ cap = cv2.VideoCapture(source,cv2.CAP_DSHOW)
+ #if cap is None or not cap.isOpened():
+ #print('Warning: unable to open video source: ', source)
+
+ if cap.isOpened():
+ #print('Opened: ',source)
+ #print('Exposure: '+ str(cap.get(cv2.CAP_PROP_EXPOSURE)))
+ #time.sleep(3)
+ cap.release()
+ cv2.destroyAllWindows()
+ open_cam = source
+ return open_cam
+ else:
+ return None
+
+def CreateAvailableCamList():
+ openCamList = []
+ for x in tqdm(range(20)):#range 20 right now to be safe
+ openCamera = TestDevice(x)
+ if openCamera is not None:
+ openCamList.append(openCamera)
+
+ return openCamList
\ No newline at end of file
diff --git a/freemocap/webcam/recordGUI.py b/freemocap/webcam/recordGUI.py
new file mode 100644
index 00000000..4b91847f
--- /dev/null
+++ b/freemocap/webcam/recordGUI.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Mar 3 12:18:46 2021
+
+@author: Rontc
+"""
+import tkinter as tk
+from tkinter import Tk,Label, Button, Frame, Listbox, Entry
+from freemocap.webcam import checkcams
+import pickle
+import datetime
+
+
+class recordGUI:
+ #this GUI detects how many cameras you have available and presents them in
+ #a listbox, and allows you to pick a task for the cameras
+ def __init__(self, master):
+ self.master = master
+ #Detect how many inputs you have available
+ available_inputs = checkcams.CreateAvailableCamList()
+ number_of_cams = len(available_inputs)
+ cam_statement = "You have " + str(number_of_cams) + " cameras available at inputs " + str(available_inputs)
+
+ self.master.title("Choose Your Cameras/Task")
+ bottom_frame = Frame(self.master, height = 100)
+ bottom_frame.pack(side = tk.BOTTOM)
+ #---creates a listbox with all the available camera choices
+ self.list_box = Listbox(self.master,selectmode = 'multiple')
+ for i in available_inputs:
+ self.list_box.insert(i,str(i))
+
+ #---setting up the task radiobutton
+ #creates the different options and the values associated with them
+ options = [("Setup", 'setup'),
+ ("Record", 'record')]
+
+ #initializes currentTask as a string variable and defaults it to setup
+ self.currentTask = tk.StringVar()
+ self.currentTask.set('setup')
+
+ #---display the detection results
+ tk.Label(self.master, text = cam_statement, justify = tk.RIGHT).pack()
+ tk.Label(self.master, text = 'Select Your Cameras', justify = tk.RIGHT).pack()
+ self.list_box.pack()
+
+ #---create the task radiobutton
+ tk.Label(self.master,
+ text="""Choose your task""",
+ justify = tk.LEFT,
+ padx = 20).pack()
+
+ for option, val in options:
+ tk.Radiobutton(self.master,
+ text=option,
+ padx = 20,
+ variable= self.currentTask,
+ value=val).pack(anchor=tk.W)
+
+
+ self.sub_button = Button(bottom_frame, text="Submit", command=self.Submit)
+ self.sub_button.pack(side = tk.RIGHT)
+
+
+
+ def Submit(self):
+ self.taskChoice = self.currentTask.get()
+ self.selected = [int(self.list_box.get(pos)) for pos in self.list_box.curselection()]
+ self.master.destroy()
+
+
+class SettingsGUI:
+ def __init__(self, master,cam_inputs,parameter_dictionary,rotation_dictionary,sessionID_in,task):
+ self.master = master
+ self.cam_inputs = cam_inputs
+ self.parameter_dictionary = parameter_dictionary
+ self.rotation_dictionary = rotation_dictionary
+ self.sessionID_in = sessionID_in
+ self.task = task
+ #check if any previous parameters exist
+ if self.parameter_dictionary is not None:
+ existing_parameters = True
+ else:
+ existing_parameters = False
+
+ #check to see if there are any stored rotations for the cameras chosen
+ rotationValues = self.RotationRetrieval(cam_inputs, rotation_dictionary)
+
+
+ #create frames to organize rotation inputs, resolution inputs, and parameter inputs
+ topFrame = Frame(self.master)
+ topFrame.pack(side = tk.TOP)
+ rotLabelFrame = Frame(self.master)
+ rotLabelFrame.pack()
+ resFrame = Frame(self.master)
+ resFrame.pack()
+ parametersFrame = Frame(self.master)
+ parametersFrame.pack()
+ bottomFrame = Frame(self.master).pack(side = tk.BOTTOM)
+
+
+ taskText = "Current Task: " + self.task
+ taskLabel = Label(topFrame,text = taskText, font = 'bold')
+ taskLabel.pack()
+ #---Rotation Options. Dynamically creates radio button options depending
+ #---on how many cameras there are
+ rotation_options = [("rotate 0",0),("rotate 90",90),("rotate 180",180),("rotate 270",270)]
+ #radVar = tk.IntVar() #Radio Button Variable
+ self.rotation_list = []
+ rotationRadioButton = {}
+ #creates radio buttons for each cam and stores the chosen rotation degree in a list
+ for cam,rotDegree in zip(self.cam_inputs,rotationValues):
+ current_rotation = tk.IntVar()
+ current_rotation.set(rotDegree)
+ self.rotation_list.append(current_rotation)
+ tk.Label(rotLabelFrame, text = "Choose rotation for Cam " + str(cam)).pack()
+ for option,degree in rotation_options:
+ rotationRadioButton[cam] = tk.Radiobutton(rotLabelFrame,text=option,value = degree,variable = current_rotation ).pack()
+
+ #---Resolution Height and Resolution Width entry
+ widthLabel = Label(resFrame, text = 'Resolution Width').pack(side = 'left')
+ self.resWidthEntry = Entry(resFrame)
+ self.resWidthEntry.pack(side = 'left')
+ heightLabel = Label(resFrame, text = 'Resolution Height').pack(side = 'left')
+ self.resHeightEntry = Entry(resFrame)
+ self.resHeightEntry.pack(side = 'left')
+
+ #---Exposure,FPS, and codec entry
+ exposureLabel = Label(parametersFrame, text = 'Exposure').pack(side= 'left')
+ self.exposureEntry = Entry(parametersFrame)
+ self.exposureEntry.pack(side = 'left')
+ FPSLabel = Label(parametersFrame, text = 'FPS').pack(side= 'left')
+ self. FPSEntry = Entry(parametersFrame)
+ self.FPSEntry.pack( side = 'left')
+ codecLabel = Label(parametersFrame, text = 'codec').pack(side = 'left')
+ self.codecEntry = Entry(parametersFrame)
+ self.codecEntry.pack(side = 'left')
+
+ #---SessionID entry- default is the date/time
+ if self.task == 'record':
+ sessionText = 'Change SessionID if desired'
+ sessionLabel = Label(parametersFrame, text = sessionText).pack()
+ self.sessionIDEntry = Entry(parametersFrame)
+ self.sessionIDEntry.pack()
+ self.sessionIDEntry.insert(0,self.sessionID_in)
+
+ #---If previous parameters were recieved, insert them into the entry boxes
+ if existing_parameters == True:
+
+ self.exposureEntry.insert(0,parameter_dictionary['exposure'])
+ self.resWidthEntry.insert(0,parameter_dictionary['resWidth'])
+ self.resHeightEntry.insert(0,parameter_dictionary['resHeight'])
+ self.codecEntry.insert(0,parameter_dictionary['codec'])
+ self.FPSEntry.insert(0,parameter_dictionary['framerate'])
+
+ #---Submit button to hit when things are finalized
+ self.sub_button = Button(bottomFrame, text="Submit", command= self.Submit)
+ self.sub_button.pack(side = tk.BOTTOM)
+
+
+ def Submit(self):
+ #get the final entries for all parameters
+ #get the rotations for each camera and save them as values of a dictionary
+ self.rotation_output = {}
+ for count,cam in enumerate(self.cam_inputs):
+ self.rotation_output[cam] = self.rotation_list[count].get()
+
+ #built a parameter dictionary by .get()-ing the values from each entry
+ self.paramDict = {'exposure':int(self.exposureEntry.get()),"resWidth":int(self.resWidthEntry.get()),"resHeight":int(self.resHeightEntry.get()),
+ 'framerate':int(self.FPSEntry.get()),'codec':str(self.codecEntry.get())}
+ #spit out the sessionID
+ if self.task =='record':
+ self.sessionID_out = self.sessionIDEntry.get()
+
+ self.master.destroy()
+ def RotationRetrieval(self,cam_inputs,rotation_dictionary):
+ #takes the input cameras and compares them to the dictionary of
+ #rotation values from the last session. For any matching cameras,
+ #grab the rotation values. Otherwise, the rotation value is None
+ rotationValues = []
+ for cam in cam_inputs:
+ if cam in rotation_dictionary.keys():
+ rotationValues.append(rotation_dictionary[cam])
+ else:
+ rotationValues.append(0)
+ return rotationValues
+
+
+
+class ProceedToRecordGUI():
+ def __init__(self, master,sessionID_in):
+ self.master = master
+ self.sessionID_in = sessionID_in
+ self.continueToRecording = ''
+ self.sessionID_out = ''
+
+ sessionText = "Proceed to recording with this sessionID?"
+ sessionLabel = Label(master,text = sessionText)
+ sessionLabel.pack(side = 'top')
+
+ self.sessionIDEntry = Entry(master)
+ self.sessionIDEntry.pack()
+ self.sessionIDEntry.insert(0,self.sessionID_in)
+
+ #master.title("Proceed to Recording?")
+ self.proceed_button = Button(text="Proceed", command=self.destroy)
+ self.stop_button = Button(text="Quit", command= self.stop)
+ self.stop_button.pack(side = tk.RIGHT)
+ self.proceed_button.pack(side = tk.RIGHT)
+
+ def stop(self):
+ self.master.destroy()
+
+ def destroy(self):
+ self.continueToRecording = True
+ self.sessionID_out = self.sessionIDEntry.get()
+ self.master.destroy()
+
+def RunGUI(sessionID_in, rotation_entry, parameter_entry):
+
+ #---Get the camera inputs and the task
+ root = tk.Tk()
+ camera_choice = recordGUI(root)
+ root.mainloop()
+
+ cam_inputs = camera_choice.selected
+ if not cam_inputs:
+ raise ValueError('No camera inputs selected')
+ task = camera_choice.taskChoice
+
+
+ #---Get all the necessary parameters
+ root = tk.Tk()
+ recording_settings = SettingsGUI(root,cam_inputs,parameter_entry,rotation_entry,sessionID_in,task)
+ root.mainloop()
+
+ #---Create the parameters we'll need to run the records
+ #rotation_input = list(test.rotation_output.values())
+
+ paramDict = recording_settings.paramDict
+
+ if task == 'record':
+ sessionID = recording_settings.sessionID_out
+ else:
+ sessionID = None
+
+ return task, cam_inputs, recording_settings.rotation_output, paramDict, sessionID
+
+def RunProceedtoRecordGUI(sessionID_in):
+ root = tk.Tk()
+ goToRecording = ProceedToRecordGUI(root,sessionID_in)
+ root.mainloop()
+
+ return goToRecording.continueToRecording,goToRecording.sessionID_out
diff --git a/freemocap/webcam/runrecordingprocess.py b/freemocap/webcam/runrecordingprocess.py
new file mode 100644
index 00000000..a30d2bd1
--- /dev/null
+++ b/freemocap/webcam/runrecordingprocess.py
@@ -0,0 +1,80 @@
+from freemocap import recordingconfig
+from freemocap.webcam import startcamrecording,timesync,videotrim
+from pathlib import Path
+import time
+import pickle
+import pandas as pd
+import numpy as np
+from tkinter import Tk,Label, Button, Frame, Listbox, Entry
+
+
+def RunCams(session,camInputs,parameterDictionary,rotationInput):
+
+
+
+ #%% Setting up recordings
+ beginTime = time.time()
+ numCams = len(camInputs) #number of cameras
+ session.numCams = numCams
+ numCamRange = range(numCams) #a range for the number of cameras that we have
+ videoNames = []
+ camIDs = []
+ for x in numCamRange: #create names for each of the initial untrimmed videos
+ singleCamID = 'Cam{}'.format(x+1)
+ camIDs.append(singleCamID) #creates IDs for each camera based on the number of cameras entered
+ singleVidName = 'raw_cam{}.mp4'.format(x+1)
+ videoNames.append(singleVidName)
+
+ #%% Starting the thread recordings for each camera
+ threads = []
+ for n in numCamRange: #starts recording video, opens threads for each camera
+ camRecordings = startcamrecording.CamRecordingThread(camIDs[n],camInputs[n],videoNames[n],session.rawVidPath,beginTime,parameterDictionary)
+ camRecordings.start()
+
+ threads.append(camRecordings)
+
+ for camRecordings in threads:
+ camRecordings.join() #make sure that one thread ending doesn't immediately end all the others (before they can dump data in a pickle file)
+
+ print('finished recordings')
+
+ timeStampList = []
+
+ for e in numCamRange: #open the saved pickle file for each camera, and add the timestamps to the dataList list
+ with open(camIDs[e], 'rb') as f:
+ camTimeList = pickle.load(f)
+ timeStampList.append(camTimeList)
+
+ timeDictionary = {}
+
+ id_and_time = zip(camIDs,timeStampList)
+
+ for cam,data in id_and_time:
+ timeDictionary[cam] = np.array(data) #create a dictionary that holds the timestamps for each camera
+
+
+ df = pd.DataFrame.from_dict(timeDictionary, orient = 'index') #create a data frame from this dictionary
+ dfT = df.transpose()
+ csvName = session.sessionID + '.csv' #create our csv filename
+ csvPath = session.sessionPath/csvName
+ dfT.to_csv(csvPath) #turn dataframe into a CSV
+
+
+ frameTable,timeTable,frameRate,resultsTable,plots = timesync.TimeSync(dfT,numCamRange,camIDs) #start the timesync process
+ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no
+ root = Tk()
+ proceed = timesync.proceedGUI(root,resultsTable,plots) #create a GUI instance called proceed
+ root.mainloop()
+
+
+ if proceed.proceed == True: #check if the proceed attribute is set to True
+ print()
+ print('Starting editing')
+ #start editing the videos
+ videotrim.VideoTrim(session,videoNames,frameTable,parameterDictionary,rotationInput,numCamRange)
+ videotrim.createCalibrationVideos(session,60,parameterDictionary)
+ print('all done')
+
+ return proceed.proceed
+
+
\ No newline at end of file
diff --git a/freemocap/webcam/startcamrecording.py b/freemocap/webcam/startcamrecording.py
new file mode 100644
index 00000000..9a433d08
--- /dev/null
+++ b/freemocap/webcam/startcamrecording.py
@@ -0,0 +1,80 @@
+import threading
+import cv2
+import time
+import pickle
+
+class CamRecordingThread(threading.Thread):
+ def __init__(self,camID,camInput,videoName,rawVidPath,beginTime,parameterDictionary):
+ threading.Thread.__init__(self)
+ self.camID = camID
+ self.camInput = camInput
+ self.videoName = videoName
+ self.rawVidPath = rawVidPath
+ self.beginTime = beginTime
+ self.parameterDictionary = parameterDictionary
+
+ def run(self):
+ print("Starting " + self.camID)
+ self.timeStamps = CamRecording(self.camID, self.camInput, self.videoName,self.rawVidPath,self.beginTime,self.parameterDictionary)
+ def getStamps(self):
+ return self.timeStamps
+#the recording function that each threaded camera object runs
+def CamRecording(camID, camInput, videoName,rawVidPath,beginTime,parameterDictionary):
+ #the flag is triggered when the user shuts down one webcam to shut down the rest.
+ #normally I'd try to avoid global variables, but in this case it's
+ #necessary, since each webcam runs as it's own object.
+ global flag
+ flag = False
+
+ cv2.namedWindow(camID) #name the preview window for the camera its showing
+ cam = cv2.VideoCapture(camInput,cv2.CAP_DSHOW) #create the video capture object
+ #if not cam.isOpened():
+ # raise RuntimeError('No camera found at input '+ str(camID))
+ #pulling out all the dictionary paramters
+ exposure = parameterDictionary.get('exposure')
+ resWidth = parameterDictionary.get('resWidth')
+ resHeight = parameterDictionary.get('resHeight')
+ framerate = parameterDictionary.get('framerate')
+ codec = parameterDictionary.get('codec')
+
+ cam.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)
+ cam.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)
+ cam.set(cv2.CAP_PROP_EXPOSURE, exposure)
+ fourcc = cv2.VideoWriter_fourcc(*codec)
+ #rawPath = filepath/'RawVideos' #creating a RawVideos folder
+ #rawPath.mkdir(parents = True, exist_ok = True)
+ width = cam.get(cv2.CAP_PROP_FRAME_WIDTH)
+ height = cam.get(cv2.CAP_PROP_FRAME_HEIGHT)
+ print('width:',width, 'height:',height)
+ saveRawVidPath = str(rawVidPath/videoName) #create a save path for each video to the RawVideos folders
+ out = cv2.VideoWriter(saveRawVidPath,fourcc, framerate, (resWidth,resHeight))
+ timeStamps = [] #holds the timestamps
+
+ if cam.isOpened():
+ success, frame = cam.read()
+ else:
+ success = False
+
+ while success: #while the camera is opened, record the data until the escape button is hit
+ if flag: #when the flag is triggered, stop recording and dump the data
+ with open(camID, 'wb') as f:
+ pickle.dump(timeStamps, f)
+ break
+ success, frame = cam.read()
+
+ cv2.imshow(camID, frame)
+ frame_sized = cv2.resize(frame,(resWidth,resHeight))
+ #frame_sized = frame
+ out.write(frame_sized)
+ timeStamps.append(time.time()-beginTime) #add each timestamp to the list
+
+ key = cv2.waitKey(20)
+ if key == 27: # exit on ESC
+ flag = True #set flag to true to shut down all other webcams
+ with open(camID, 'wb') as f:
+ pickle.dump(timeStamps, f) #dump the data
+ break
+ cv2.destroyWindow(camID)
+ return timeStamps
+
+#this is how we sync our time frames, based on our recorded timestamps
\ No newline at end of file
diff --git a/freemocap/webcam/timesync.py b/freemocap/webcam/timesync.py
new file mode 100644
index 00000000..814fe391
--- /dev/null
+++ b/freemocap/webcam/timesync.py
@@ -0,0 +1,187 @@
+import numpy as np
+import pandas as pd
+import sys
+from matplotlib.figure import Figure
+from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
+import tkinter as tk
+from tkinter import Tk,Label, Button, Frame
+
+def TimeSync(timeStampData,numCamRange,camNames):
+
+ def CloseNeighb(camera,point):
+ closestPoint = (np.abs(camera - point)).argmin()
+ return closestPoint
+
+
+ #this section auto-finds the start and end points for our master timeline to the nearest second
+ masterTimelineBegin = np.ceil(max(timeStampData.iloc[0]))
+ lastPoints = []
+ for name in camNames:
+ cameraLastPoint = timeStampData[name][timeStampData[name].last_valid_index()] #find the last non-nan value in each camera
+ lastPoints.append(cameraLastPoint)
+
+ masterTimelineEnd = np.floor(min(lastPoints)) #find where the fastest camera ended, round down, use that as the end point
+
+
+ #print('end',endList)
+ print('intervals:',masterTimelineBegin,masterTimelineEnd)
+
+
+ totalFrameIntvl= [] #list for storing frame rates for each camera
+
+
+ n = 0; #counter for going through camera names
+ for x in numCamRange:
+ currentCam = timeStampData[camNames[x]]
+ #finds the closest point in each camera to the start and end points of our timeline
+ currentCamStart = CloseNeighb(currentCam,masterTimelineBegin)
+ currentCamEnd = CloseNeighb(currentCam,masterTimelineEnd)
+ print("using frames:", currentCamStart,"-",currentCamEnd,"for",camNames[n])
+ n +=1
+ currentCamTimeline = currentCam[currentCamStart:currentCamEnd] #grab the times from start to finish for each camera
+ currentCamFrameInterval = np.mean(np.diff(currentCamTimeline)) #calculate the interval between each frame and take the mean
+ totalFrameIntvl.append(currentCamFrameInterval) #add interval to list
+ #print(camNames[x],currentCamStart,currentCamEnd)
+ totalAverageIntvl = np.mean(totalFrameIntvl) #find the total average interval across all cameras
+ masterTimeline = np.arange(masterTimelineBegin,masterTimelineEnd,totalAverageIntvl) #build a master timeline with the average interval
+
+ #now we start the syncing process
+
+ frameList = masterTimeline #start a list of frames, with the first row being our master timeline
+ timeList = masterTimeline #start a list of timestamps,with the first row being our master timeline
+
+ delNum= []; #stores number of deleted frames/camera
+ bufNum= []; #stores number of buffered frames/camera
+ bufPercentList = []; #stores percentage of deleted frames/camera
+ delPercentList = []; #stores percentage of buffered frames/camera
+
+ count = 0 #Keeps track of what frame we're on (I think?)
+ n = 0; #I don't remember why I did this but I'm sure I'll figure it out later
+
+ for y in numCamRange:
+ thisCam = timeStampData[camNames[y]]
+ camTimes = []; #stored times
+ camFrames =[]; #stored frames
+
+ count += 1
+ #--------------Adjust each camera to start at the first point in the master timeline
+ beginFrame = CloseNeighb(thisCam,masterTimeline[0])
+ timeDif = masterTimeline[0] - thisCam[beginFrame]
+ thisCam = thisCam + timeDif
+
+ for z in masterTimeline: #for each point in the master timeline
+ closestFrame = CloseNeighb(thisCam, z) #find the closest frame in this camera to each point of the master timeline
+ camFrames.append(closestFrame) #add that frame to the list
+ camTimes.append(thisCam[closestFrame]) #find the time corresponding to this frame
+
+ print("starting detection:",camNames[n]) #now to start finding deleted/buffered slides
+ frameList = np.column_stack((frameList,camFrames)) #update our framelist with our new frames
+ timeList = np.column_stack((timeList,camTimes)) #update our timelist
+
+
+ #start counters for the number of buffered and deleted slides
+ bufCount = 0;
+ delCount = 0;
+
+ for i in range(0,len(camFrames)-1):
+ distanceBetweenFrames = camFrames[i+1] - camFrames[i] #find the distance between adjacent frames
+ if distanceBetweenFrames == 1: #these frames are consecutive, do nothing
+ None
+ elif distanceBetweenFrames == 0: #we have a buffered slide
+ bufCount += 1
+ #this section looks at our two timepoints, and finds the which point on the master timeline is closest
+ frame1 = abs(masterTimeline[i]-camTimes[i])
+ frame2 = abs(masterTimeline[i+1]-camTimes[i])
+ #finds which frame is closest, and sets the other frame as a buffer (indicated by the -1)
+ if frame1>frame2:
+ frameList[i,count] = -1
+ timeList[i,count] = -1
+ else:
+ frameList[i+1,count] = -1;
+ timeList[i+1,count] = -1;
+ elif distanceBetweenFrames > 1: #deletion
+ delCount += 1
+ else:
+
+ print("something else happened")
+ #update and calculate our percentages and numbers for buffers/deletions
+ delNum.append(round(delCount,1))
+ bufNum.append(round(bufCount,1))
+ #print(bufCount,len(frameList))
+ bufPercent = (bufCount/len(frameList))*100
+ delPercent = (delCount/len(frameList))*100
+ bufPercentList.append(round(bufPercent,2))
+ delPercentList.append(round(delPercent,2))
+
+ #print(delNum,bufNum,bufPercentList,delPercentList)
+ #print("deleted frames:",delCount)
+ #print("buffered frames:",bufCount)
+ n +=1
+
+ #create our data frame for both times and frames
+ frameTable = pd.DataFrame(frameList)
+ columnNames = ['Master Timeline'] + camNames
+ frameTable.columns = columnNames
+ timeTable = pd.DataFrame(timeList)
+ timeTable.columns = columnNames
+ totalFrameRate = [round(1/intvl,1) for intvl in totalFrameIntvl]
+ frameRate = 1/totalAverageIntvl #calculates our framerate
+ results = {'Cam':camNames,'#Del':delNum,'%Del':delPercentList,'#Buf':bufNum,'%Buf':bufPercentList,'FPS':totalFrameRate}
+ resultTable = pd.DataFrame(results,columns = ['Cam','#Del','%Del','#Buf','%Buf','FPS'])
+
+ #============================================== plot data
+ differenceFrame = timeStampData.diff(axis = 0)
+ fpsFrame = differenceFrame.pow(-1)
+ fig = Figure(figsize = [10,10])
+ fig.patch.set_facecolor('#F0F0F0')
+ a = fig.add_subplot(221)
+ a.set_xlabel('Frame')
+ a.set_ylabel('Time(s)')
+ timeStampData.plot(ax = a, title = 'Camera timestamps')
+ b = fig.add_subplot(222)
+ b.set_xlabel('Frame')
+ b.set_ylabel('Frame duration (s)')
+ differenceFrame.plot(ax = b, marker = '.', linestyle = 'none',title = 'Camera frame duration')
+ c = fig.add_subplot(223)
+ c.set_xlabel('Time (s)')
+ differenceFrame.plot.hist(ax = c, bins=6,alpha = .5, xlim = [0,.1], title = 'Frame interval distribution (s)')
+ d = fig.add_subplot(224)
+ d.set_xlabel('Frames per second')
+ fpsFrame.plot.hist(ax = d, bins=6,alpha = .5, xlim = [0,40], title = 'Frames per second distribution')
+ return frameTable,timeTable,frameRate,resultTable,fig
+#function to trim our videos
+
+class proceedGUI:
+ def __init__(self, master, results, figure):
+ self.master = master
+ self.results = results
+ self.figure = figure
+ self.proceed = ''
+ master.title("Choose File Path")
+
+
+ bottom_frame = Frame(self.master, height = 1000)
+ bottom_frame.pack(side = tk.LEFT)
+ self.label = Label(bottom_frame, text= results.to_string(index = False))
+ self.label.pack(side = tk.TOP)
+ self.go_button = Button(bottom_frame, text="Proceed", command=self.destroy)
+ self.stop_button = Button(bottom_frame, text="Quit", command= self.stop)
+
+ self.canvas = FigureCanvasTkAgg(self.figure, master=self.master)
+ self.canvas.draw()
+ self.canvas.get_tk_widget().pack(side=tk.LEFT, fill="both", expand=1)
+
+ self.stop_button.pack(side = tk.RIGHT)
+ self.go_button.pack(side = tk.RIGHT)
+
+
+
+
+
+ def stop(self):
+ self.master.destroy()
+ sys.exit("Quitting Program")
+
+ def destroy(self):
+ self.proceed = True
+ self.master.destroy()
\ No newline at end of file
diff --git a/freemocap/webcam/videotrim.py b/freemocap/webcam/videotrim.py
new file mode 100644
index 00000000..4f4e8924
--- /dev/null
+++ b/freemocap/webcam/videotrim.py
@@ -0,0 +1,80 @@
+from rich.progress import track
+import cv2
+import imutils
+import numpy as np
+import os
+
+def VideoTrim(session,vidList,ft,parameterDictionary,rotationState,numCamRange):
+ camList = list(ft.columns[1:len(vidList)+1]) #grab the camera identifiers from the data frame
+ resWidth = parameterDictionary.get('resWidth')
+ resHeight = parameterDictionary.get('resHeight')
+ framerate = parameterDictionary.get('framerate')
+ trueWidth = resWidth #creating true resWidth and resHeight in case we need to rotate our images
+ trueHeight = resHeight
+ codec = parameterDictionary.get('codec')
+
+ for vid,cam,camNum in zip(vidList,camList,numCamRange): #iterate in parallel through camera identifiers and matching videos
+ print('Editing '+cam+' from ' +vid)
+ cap = cv2.VideoCapture(str(session.rawVidPath/vid)) #initialize OpenCV capture
+ frameTable = ft[cam] #grab the frames needed for that camera
+ success, image = cap.read() #start reading frames
+ fourcc = cv2.VideoWriter_fourcc(*codec)
+ saveName = session.sessionID +'_synced_' + cam + '.mp4'
+
+
+ saveSyncedVidPath = str(session.syncedVidPath/saveName) #create an output path for the function
+ if rotationState[camNum] == 90 or rotationState[camNum] == 270:
+ #if we are rotating to these angles, we need to temporarily swap our resWidth and resHeight
+ tempHeight = resHeight
+ resHeight = resWidth
+ resWidth = tempHeight
+
+ out = cv2.VideoWriter(saveSyncedVidPath, fourcc, framerate, (resWidth,resHeight)) #change resolution as needed
+ for frame in track(frameTable): #start looking through the frames we need
+ if frame == -1: #this is a buffer frame
+ blankFrame = np.zeros_like(image) #create a blank frame
+ if rotationState[camNum] is not None:
+ blankFrame = imutils.rotate_bound(blankFrame, angle= rotationState[camNum])
+ image = cv2.resize(image,(resWidth,resHeight))
+ out.write(blankFrame) #write that frame to the video
+ else:
+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need
+ success, image = cap.read()
+ if rotationState[camNum] is not None:
+ image = imutils.rotate_bound(image, angle= rotationState[camNum])
+ image = cv2.resize(image,(resWidth,resHeight))
+ out.write(image)
+ resWidth = trueWidth
+ resHeight = trueHeight
+ cap.release()
+ out.release()
+ print('Saved '+ saveSyncedVidPath)
+ print()
+
+def createCalibrationVideos(session,calVideoFrameLength,parameterDictionary):
+ vidList = os.listdir(session.syncedVidPath)
+ framelist = list(range(calVideoFrameLength))
+ codec = parameterDictionary.get('codec')
+ for count,vid in enumerate(vidList,start=1):
+ cam_name = "Cam{}".format(count)
+ cap = cv2.VideoCapture(str(session.syncedVidPath/vid))
+ fourcc = cv2.VideoWriter_fourcc(*codec)
+
+ #grab resolution parameters from the videos
+ resWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ resHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ framerate = int(cap.get(cv2.CAP_PROP_FPS))
+
+ saveName = session.sessionID +'_trimmed_' + cam_name + '.mp4' #create a name for the trimmed video
+ saveCalVidPath = str(session.calVidPath/saveName) #create an output path for the function
+
+ success, image = cap.read() #start reading frames
+
+ out = cv2.VideoWriter(saveCalVidPath, fourcc, framerate, (resWidth,resHeight))
+ print('Trimming ' + cam_name)
+ for frame in track(framelist):
+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need
+ success, image = cap.read()
+ out.write(image)
+ cap.release()
+ out.release()
\ No newline at end of file
diff --git a/freemocap/webcam/webcam.py b/freemocap/webcam/webcam.py
new file mode 100644
index 00000000..31ced8fe
--- /dev/null
+++ b/freemocap/webcam/webcam.py
@@ -0,0 +1,632 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Nov 30 12:48:14 2020
+
+@author: Rontc
+"""
+
+import cv2
+import threading
+import pickle
+import time
+import numpy as np
+import pandas as pd
+import tkinter as tk
+from tkinter import Tk,Label, Button, Frame
+#import tkMessageBox
+from tqdm import tqdm
+import imutils
+import sys
+from matplotlib.figure import Figure
+from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
+from omc import recordingconfig
+from pathlib import Path
+
+global beginTime
+
+
+#create a camera object that can be threaded
+class CamThread(threading.Thread):
+ def __init__(self,camID,camInput,videoName,rawVidPath,beginTime,parameterDictionary):
+ threading.Thread.__init__(self)
+ self.camID = camID
+ self.camInput = camInput
+ self.videoName = videoName
+ self.rawVidPath = rawVidPath
+ self.beginTime = beginTime
+ self.parameterDictionary = parameterDictionary
+
+ def run(self):
+ print("Starting " + self.camID)
+ self.timeStamps = CamPreview(self.camID, self.camInput, self.videoName,self.rawVidPath,self.beginTime,self.parameterDictionary)
+ def getStamps(self):
+ return self.timeStamps
+#the recording function that each threaded camera object runs
+def CamPreview(camID, camInput, videoName,rawVidPath,beginTime,parameterDictionary):
+ #the flag is triggered when the user shuts down one webcam to shut down the rest.
+ #normally I'd try to avoid global variables, but in this case it's
+ #necessary, since each webcam runs as it's own object.
+ global flag
+ flag = False
+
+ cv2.namedWindow(camID) #name the preview window for the camera its showing
+ cam = cv2.VideoCapture(camInput,cv2.CAP_DSHOW) #create the video capture object
+ #if not cam.isOpened():
+ # raise RuntimeError('No camera found at input '+ str(camID))
+ #pulling out all the dictionary paramters
+ exposure = parameterDictionary.get('exposure')
+ resWidth = parameterDictionary.get('resWidth')
+ resHeight = parameterDictionary.get('resHeight')
+ framerate = parameterDictionary.get('framerate')
+ codec = parameterDictionary.get('codec')
+
+ cam.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)
+ cam.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)
+ cam.set(cv2.CAP_PROP_EXPOSURE, exposure)
+ fourcc = cv2.VideoWriter_fourcc(*codec)
+ #rawPath = filepath/'RawVideos' #creating a RawVideos folder
+ #rawPath.mkdir(parents = True, exist_ok = True)
+ width = cam.get(cv2.CAP_PROP_FRAME_WIDTH)
+ height = cam.get(cv2.CAP_PROP_FRAME_HEIGHT)
+ print('width:',width, 'height:',height)
+ saveRawVidPath = str(rawVidPath/videoName) #create a save path for each video to the RawVideos folders
+ out = cv2.VideoWriter(saveRawVidPath,fourcc, framerate, (resWidth,resHeight))
+ timeStamps = [] #holds the timestamps
+
+ if cam.isOpened():
+ success, frame = cam.read()
+ else:
+ success = False
+
+ while success: #while the camera is opened, record the data until the escape button is hit
+ if flag: #when the flag is triggered, stop recording and dump the data
+ with open(camID, 'wb') as f:
+ pickle.dump(timeStamps, f)
+ break
+ success, frame = cam.read()
+
+ cv2.imshow(camID, frame)
+ frame_sized = cv2.resize(frame,(resWidth,resHeight))
+ #frame_sized = frame
+ out.write(frame_sized)
+ timeStamps.append(time.time()-beginTime) #add each timestamp to the list
+
+ key = cv2.waitKey(20)
+ if key == 27: # exit on ESC
+ flag = True #set flag to true to shut down all other webcams
+ with open(camID, 'wb') as f:
+ pickle.dump(timeStamps, f) #dump the data
+ break
+ cv2.destroyWindow(camID)
+ return timeStamps
+
+#this is how we sync our time frames, based on our recorded timestamps
+class proceedGUI:
+ def __init__(self, master, results, figure):
+ self.master = master
+ self.results = results
+ self.figure = figure
+ master.title("Choose File Path")
+
+
+ bottom_frame = Frame(self.master, height = 1000)
+ bottom_frame.pack(side = tk.LEFT)
+ self.label = Label(bottom_frame, text= results.to_string(index = False))
+ self.label.pack(side = tk.TOP)
+ self.go_button = Button(bottom_frame, text="Proceed", command=self.destroy)
+ self.stop_button = Button(bottom_frame, text="Quit", command= self.stop)
+
+ self.canvas = FigureCanvasTkAgg(self.figure, master=self.master)
+ self.canvas.draw()
+ self.canvas.get_tk_widget().pack(side=tk.LEFT, fill="both", expand=1)
+
+ self.stop_button.pack(side = tk.RIGHT)
+ self.go_button.pack(side = tk.RIGHT)
+
+
+
+
+
+ def stop(self):
+ self.master.destroy()
+ sys.exit("Quitting Program")
+
+ def destroy(self):
+ self.master.destroy()
+
+def TimeSync(df,numCamRange,camNames):
+
+ def CloseNeighb(camera,point):
+ closestPoint = (np.abs(camera - point)).argmin()
+ return closestPoint
+
+ newFrame = df #read our CSV file
+
+
+
+ #this section auto-finds the start and end points for our master timeline to the nearest second
+ masterTimelineBegin = np.ceil(max(newFrame.iloc[0]))
+ lastPoints = []
+ for name in camNames:
+ cameraLastPoint = newFrame[name][newFrame[name].last_valid_index()] #find the last non-nan value in each camera
+ lastPoints.append(cameraLastPoint)
+
+ masterTimelineEnd = np.floor(min(lastPoints)) #find where the fastest camera ended, round down, use that as the end point
+
+
+ #print('end',endList)
+ print('intervals:',masterTimelineBegin,masterTimelineEnd)
+
+
+ totalFrameIntvl= [] #list for storing frame rates for each camera
+
+
+ n = 0; #counter for going through camera names
+ for x in numCamRange:
+ currentCam = newFrame[camNames[x]]
+ #finds the closest point in each camera to the start and end points of our timeline
+ currentCamStart = CloseNeighb(currentCam,masterTimelineBegin)
+ currentCamEnd = CloseNeighb(currentCam,masterTimelineEnd)
+ print("using frames:", currentCamStart,"-",currentCamEnd,"for",camNames[n])
+ n +=1
+ currentCamTimeline = currentCam[currentCamStart:currentCamEnd] #grab the times from start to finish for each camera
+ currentCamFrameInterval = np.mean(np.diff(currentCamTimeline)) #calculate the interval between each frame and take the mean
+ totalFrameIntvl.append(currentCamFrameInterval) #add interval to list
+ #print(camNames[x],currentCamStart,currentCamEnd)
+ totalAverageIntvl = np.mean(totalFrameIntvl) #find the total average interval across all cameras
+ masterTimeline = np.arange(masterTimelineBegin,masterTimelineEnd,totalAverageIntvl) #build a master timeline with the average interval
+
+ #now we start the syncing process
+
+ frameList = masterTimeline #start a list of frames, with the first row being our master timeline
+ timeList = masterTimeline #start a list of timestamps,with the first row being our master timeline
+
+ delNum= []; #stores number of deleted frames/camera
+ bufNum= []; #stores number of buffered frames/camera
+ bufPercentList = []; #stores percentage of deleted frames/camera
+ delPercentList = []; #stores percentage of buffered frames/camera
+
+ count = 0 #Keeps track of what frame we're on (I think?)
+ n = 0; #I don't remember why I did this but I'm sure I'll figure it out later
+
+ for y in numCamRange:
+ thisCam = newFrame[camNames[y]]
+ camTimes = []; #stored times
+ camFrames =[]; #stored frames
+
+ count += 1
+ #--------------Adjust each camera to start at the first point in the master timeline
+ beginFrame = CloseNeighb(thisCam,masterTimeline[0])
+ timeDif = masterTimeline[0] - thisCam[beginFrame]
+ thisCam = thisCam + timeDif
+
+ for z in masterTimeline: #for each point in the master timeline
+ closestFrame = CloseNeighb(thisCam, z) #find the closest frame in this camera to each point of the master timeline
+ camFrames.append(closestFrame) #add that frame to the list
+ camTimes.append(thisCam[closestFrame]) #find the time corresponding to this frame
+
+ print("starting detection:",camNames[n]) #now to start finding deleted/buffered slides
+ frameList = np.column_stack((frameList,camFrames)) #update our framelist with our new frames
+ timeList = np.column_stack((timeList,camTimes)) #update our timelist
+
+
+ #start counters for the number of buffered and deleted slides
+ bufCount = 0;
+ delCount = 0;
+
+ for i in range(0,len(camFrames)-1):
+ distanceBetweenFrames = camFrames[i+1] - camFrames[i] #find the distance between adjacent frames
+ if distanceBetweenFrames == 1: #these frames are consecutive, do nothing
+ None
+ elif distanceBetweenFrames == 0: #we have a buffered slide
+ bufCount += 1
+ #this section looks at our two timepoints, and finds the which point on the master timeline is closest
+ frame1 = abs(masterTimeline[i]-camTimes[i])
+ frame2 = abs(masterTimeline[i+1]-camTimes[i])
+ #finds which frame is closest, and sets the other frame as a buffer (indicated by the -1)
+ if frame1>frame2:
+ frameList[i,count] = -1
+ timeList[i,count] = -1
+ else:
+ frameList[i+1,count] = -1;
+ timeList[i+1,count] = -1;
+ elif distanceBetweenFrames > 1: #deletion
+ delCount += 1
+ else:
+
+ print("something else happened")
+ #update and calculate our percentages and numbers for buffers/deletions
+ delNum.append(round(delCount,1))
+ bufNum.append(round(bufCount,1))
+ #print(bufCount,len(frameList))
+ bufPercent = (bufCount/len(frameList))*100
+ delPercent = (delCount/len(frameList))*100
+ bufPercentList.append(round(bufPercent,2))
+ delPercentList.append(round(delPercent,2))
+
+ #print(delNum,bufNum,bufPercentList,delPercentList)
+ #print("deleted frames:",delCount)
+ #print("buffered frames:",bufCount)
+ n +=1
+
+ #create our data frame for both times and frames
+ frameTable = pd.DataFrame(frameList)
+ columnNames = ['Master Timeline'] + camNames
+ frameTable.columns = columnNames
+ timeTable = pd.DataFrame(timeList)
+ timeTable.columns = columnNames
+ totalFrameRate = [round(1/intvl,1) for intvl in totalFrameIntvl]
+ frameRate = 1/totalAverageIntvl #calculates our framerate
+ results = {'Cam':camNames,'#Del':delNum,'%Del':delPercentList,'#Buf':bufNum,'%Buf':bufPercentList,'FPS':totalFrameRate}
+ resultTable = pd.DataFrame(results,columns = ['Cam','#Del','%Del','#Buf','%Buf','FPS'])
+
+ #============================================== plot data
+ differenceFrame = newFrame.diff(axis = 0)
+ fpsFrame = differenceFrame.pow(-1)
+ fig = Figure(figsize = [10,10])
+ fig.patch.set_facecolor('#F0F0F0')
+ a = fig.add_subplot(221)
+ a.set_xlabel('Frame')
+ a.set_ylabel('Time(s)')
+ newFrame.plot(ax = a, title = 'Camera timestamps')
+ b = fig.add_subplot(222)
+ b.set_xlabel('Frame')
+ b.set_ylabel('Frame duration (s)')
+ differenceFrame.plot(ax = b, marker = '.', linestyle = 'none',title = 'Camera frame duration')
+ c = fig.add_subplot(223)
+ c.set_xlabel('Time (s)')
+ differenceFrame.plot.hist(ax = c, bins=6,alpha = .5, xlim = [0,.1], title = 'Frame interval distribution (s)')
+ d = fig.add_subplot(224)
+ d.set_xlabel('Frames per second')
+ fpsFrame.plot.hist(ax = d, bins=6,alpha = .5, xlim = [0,40], title = 'Frames per second distribution')
+ return frameTable,timeTable,frameRate,resultTable,fig
+#function to trim our videos
+
+
+def VideoEdit(rawVidPath,syncedVidPath, vidList,sessionName,ft,parameterDictionary,rotationState,numCamRange):
+ camList = list(ft.columns[1:len(vidList)+1]) #grab the camera identifiers from the data frame
+ resWidth = parameterDictionary.get('resWidth')
+ resHeight = parameterDictionary.get('resHeight')
+ framerate = parameterDictionary.get('framerate')
+ trueWidth = resWidth
+ trueHeight = resHeight
+ codec = parameterDictionary.get('codec')
+ for vid,cam,camNum in zip(vidList,camList,numCamRange): #iterate in parallel through camera identifiers and matching videos
+ print('Editing '+cam+' from ' +vid)
+ #print(cam+'_'+out_path)
+ #rawPath = filepath/'RawVideos'
+ cap = cv2.VideoCapture(str(rawVidPath/vid)) #initialize OpenCV capture
+ frameTable = ft[cam] #grab the frames needed for that camera
+ success, image = cap.read() #start reading frames
+ fourcc = cv2.VideoWriter_fourcc(*codec)
+ saveName = sessionName +'_synced_' + cam + '.mp4'
+ #syncedPath = filepath/'SyncedVideos'
+ #syncedPath.mkdir(parents = True, exist_ok = True)
+
+
+ saveSyncedVidPath = str(syncedVidPath/saveName) #create an output path for the function
+ if rotationState[camNum] == 90 or rotationState[camNum] == 270:
+ tempHeight = resHeight
+ resHeight = resWidth
+ resWidth = tempHeight
+ #print(rotationState[camNum], resHeight, resWidth)
+
+ out = cv2.VideoWriter(saveSyncedVidPath, fourcc, framerate, (resWidth,resHeight)) #change resolution as needed
+ for frame in tqdm(frameTable, leave = True): #start looking through the frames we need
+ if frame == -1: #this is a buffer frame
+ blankFrame = np.zeros_like(image) #create a blank frame
+ if rotationState[camNum] is not None:
+ blankFrame = imutils.rotate_bound(blankFrame, angle= rotationState[camNum])
+ #image = cv2.rotate(image,rotateCode = rotationState[camNum])
+ image = cv2.resize(image,(resWidth,resHeight))
+ out.write(blankFrame) #write that frame to the video
+ else:
+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need
+ success, image = cap.read()
+ if rotationState[camNum] is not None:
+ image = imutils.rotate_bound(image, angle= rotationState[camNum])
+ #image = cv2.rotate(image,rotateCode = rotationState[camNum])
+ image = cv2.resize(image,(resWidth,resHeight))
+ out.write(image)
+ resWidth = trueWidth
+ resHeight = trueHeight
+ cap.release()
+ out.release()
+ print('Saved '+ saveSyncedVidPath)
+ print()
+
+
+#function to run all these above functions together
+def RunCams(camInputs,config_yaml_path,sessionName,parameterDictionary,rotationInput):
+ csvName = sessionName + '.csv' #create our csv filename
+
+ config_settings = recordingconfig.load_config_yaml(config_yaml_path)
+ rawVidPath = Path(config_settings['Paths']['rawVidPath'])
+ syncedVidPath = Path(config_settings['Paths']['syncedVidPath'])
+
+
+ beginTime = time.time()
+ numCams = len(camInputs) #number of cameras
+ numCamRange = range(numCams) #a range for the number of cameras that we have
+ videoNames = []
+ camIDs = []
+ for x in numCamRange: #create names for each of the initial untrimmed videos
+ singleCamID = 'Cam{}'.format(x+1)
+ camIDs.append(singleCamID) #creates IDs for each camera based on the number of cameras entered
+ singleVidName = 'raw_cam{}.mp4'.format(x+1)
+ videoNames.append(singleVidName)
+
+ threads = []
+
+ for n in numCamRange: #starts recording video, opens threads for each camera
+ camRecordings = CamThread(camIDs[n],camInputs[n],videoNames[n],rawVidPath,beginTime,parameterDictionary)
+ camRecordings.start()
+
+ threads.append(camRecordings)
+
+ for camRecordings in threads:
+ camRecordings.join() #make sure that one thread ending doesn't immediately end all the others (before they can dump data in a pickle file)
+
+ print('finished')
+
+ timeStampList = []
+
+ for e in numCamRange: #open the saved pickle file for each camera, and add the timestamps to the dataList list
+ with open(camIDs[e], 'rb') as f:
+ camTimeList = pickle.load(f)
+ timeStampList.append(camTimeList)
+
+ timeDictionary = {} #our dictionary
+
+ id_and_time = zip(camIDs,timeStampList)
+
+ for cam,data in id_and_time:
+ timeDictionary[cam] = np.array(data) #create a dictionary that holds the timestamps for each camera
+
+
+ df = pd.DataFrame.from_dict(timeDictionary, orient = 'index') #create a data frame from this dictionary
+ dfT = df.transpose()
+ csvPath = Path(config_settings['Paths']['sessionPath'])/csvName
+ dfT.to_csv(csvPath) #turn dataframe into a CSV
+
+
+ frameTable,timeTable,frameRate,resultsTable,plots = TimeSync(dfT,numCamRange,camIDs) #start the timesync process
+ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no
+ root = Tk()
+
+ proceed = proceedGUI(root,resultsTable,plots)
+ root.mainloop()
+
+
+ print()
+ print('Starting editing')
+ #start editing the videos
+ VideoEdit(rawVidPath,syncedVidPath,videoNames,sessionName,frameTable,parameterDictionary,rotationInput,numCamRange)
+
+
+ print('all done')
+
+def TestDevice(source):
+ cap = cv2.VideoCapture(source,cv2.CAP_DSHOW)
+ #if cap is None or not cap.isOpened():
+ #print('Warning: unable to open video source: ', source)
+
+ if cap.isOpened():
+ #print('Opened: ',source)
+ #print('Exposure: '+ str(cap.get(cv2.CAP_PROP_EXPOSURE)))
+ #time.sleep(3)
+ cap.release()
+ cv2.destroyAllWindows()
+ open_cam = source
+ return open_cam
+ else:
+ return None
+
+def CheckCams():
+ openCamList = []
+ for x in tqdm(range(20)):#range 20 right now to be safe
+ openCamera = TestDevice(x)
+ if openCamera is not None:
+ openCamList.append(openCamera)
+
+ return openCamList
+
+class VideoSetup(threading.Thread):
+ def __init__(self, camID,parameterDictionary,rotNum):
+ self.camID = camID
+ self.parameterDictionary = parameterDictionary
+ self.rotNum = rotNum
+ threading.Thread.__init__(self)
+ def run(self):
+ #print("Starting " + self.previewName)
+ self.record(self.parameterDictionary,self.rotNum)
+ def record(self,parameterDictionary,rotNum):
+ exposure = parameterDictionary.get('exposure')
+ resWidth = parameterDictionary.get('resWidth')
+ resHeight = parameterDictionary.get('resHeight')
+ camName = "Camera" + str(self.camID)
+
+ cv2.namedWindow(camName)
+ cap = cv2.VideoCapture(self.camID,cv2.CAP_DSHOW)
+ cap.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)
+ cap.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)
+ cap.set(cv2.CAP_PROP_EXPOSURE, exposure)
+ #print('exposure',cap.get(cv2.CAP_PROP_EXPOSURE))
+ #if not cap.isOpened():
+ # raise RuntimeError('No camera found at input '+ str(self.camID))
+ while True:
+ ret1,frame1 = cap.read()
+ if ret1 ==True:
+ if rotNum is not None:
+ frame1 = imutils.rotate_bound(frame1, angle= rotNum)
+ cv2.imshow(camName,frame1)
+ if cv2.waitKey(1) & 0xFF== 27:
+ # == ord('q') for q
+ break
+
+ else:
+ break
+ cv2.destroyWindow(camName)
+
+
+#function to trim our videos
+
+
+
+
+def DebugTime(sessionName,recordPath):
+
+ def CloseNeighb(camera,point):
+ closestPoint = (np.abs(camera - point)).argmin()
+ return closestPoint
+ csvName = sessionName + '.csv'
+ csvPath = recordPath/csvName
+ newFrame = pd.read_csv (csvPath)
+ newFrame = newFrame.iloc[:,1:]
+ camNames = list(newFrame.columns)
+ numCamRange = range(len(camNames))
+
+
+
+
+ #this section auto-finds the start and end points for our master timeline to the nearest second
+ masterTimelineBegin = np.ceil(max(newFrame.iloc[0]))
+ lastPoints = []
+ for name in camNames:
+ cameraLastPoint = newFrame[name][newFrame[name].last_valid_index()] #find the last non-nan value in each camera
+ lastPoints.append(cameraLastPoint)
+
+ masterTimelineEnd = np.floor(min(lastPoints)) #find where the fastest camera ended, round down, use that as the end point
+
+
+ #print('end',endList)
+ print('intervals:',masterTimelineBegin,masterTimelineEnd)
+
+
+ totalFrameIntvl= [] #list for storing frame rates for each camera
+
+
+ n = 0; #counter for going through camera names
+ for x in numCamRange:
+ currentCam = newFrame[camNames[x]]
+ #finds the closest point in each camera to the start and end points of our timeline
+ currentCamStart = CloseNeighb(currentCam,masterTimelineBegin)
+ currentCamEnd = CloseNeighb(currentCam,masterTimelineEnd)
+ print("using frames:", currentCamStart,"-",currentCamEnd,"for",camNames[n])
+ print(n)
+ n +=1
+ currentCamTimeline = currentCam[currentCamStart:currentCamEnd] #grab the times from start to finish for each camera
+ currentCamFrameInterval = np.mean(np.diff(currentCamTimeline)) #calculate the interval between each frame and take the mean
+ totalFrameIntvl.append(currentCamFrameInterval) #add interval to list
+ #print(camNames[x],currentCamStart,currentCamEnd)
+
+ totalAverageIntvl = np.mean(totalFrameIntvl) #find the total average interval across all cameras
+ masterTimeline = np.arange(masterTimelineBegin,masterTimelineEnd,totalAverageIntvl) #build a master timeline with the average interval
+
+ #now we start the syncing process
+
+ frameList = masterTimeline #start a list of frames, with the first row being our master timeline
+ timeList = masterTimeline #start a list of timestamps,with the first row being our master timeline
+
+ delNum= []; #stores number of deleted frames/camera
+ bufNum= []; #stores number of buffered frames/camera
+ bufPercentList = []; #stores percentage of deleted frames/camera
+ delPercentList = []; #stores percentage of buffered frames/camera
+
+ count = 0 #Keeps track of what frame we're on (I think?)
+ n = 0; #I don't remember why I did this but I'm sure I'll figure it out later
+
+ for y in numCamRange:
+ thisCam = newFrame[camNames[y]]
+ camTimes = []; #stored times
+ camFrames =[]; #stored frames
+
+ count += 1
+ #--------------Adjust each camera to start at the first point in the master timeline
+ beginFrame = CloseNeighb(thisCam,masterTimeline[0])
+ timeDif = masterTimeline[0] - thisCam[beginFrame]
+ thisCam = thisCam + timeDif
+
+ for z in masterTimeline: #for each point in the master timeline
+ closestFrame = CloseNeighb(thisCam, z) #find the closest frame in this camera to each point of the master timeline
+ camFrames.append(closestFrame) #add that frame to the list
+ camTimes.append(thisCam[closestFrame]) #find the time corresponding to this frame
+
+ print("starting detection:",camNames[n]) #now to start finding deleted/buffered slides
+ frameList = np.column_stack((frameList,camFrames)) #update our framelist with our new frames
+ timeList = np.column_stack((timeList,camTimes)) #update our timelist
+
+
+ #start counters for the number of buffered and deleted slides
+ bufCount = 0;
+ delCount = 0;
+
+ for i in range(0,len(camFrames)-1):
+ distanceBetweenFrames = camFrames[i+1] - camFrames[i] #find the distance between adjacent frames
+ if distanceBetweenFrames == 1: #these frames are consecutive, do nothing
+ None
+ elif distanceBetweenFrames == 0: #we have a buffered slide
+ bufCount += 1
+ #this section looks at our two timepoints, and finds the which point on the master timeline is closest
+ frame1 = abs(masterTimeline[i]-camTimes[i])
+ frame2 = abs(masterTimeline[i+1]-camTimes[i])
+ #finds which frame is closest, and sets the other frame as a buffer (indicated by the -1)
+ if frame1>frame2:
+ frameList[i,count] = -1
+ timeList[i,count] = -1
+ else:
+ frameList[i+1,count] = -1;
+ timeList[i+1,count] = -1;
+ elif distanceBetweenFrames > 1: #deletion
+ delCount += 1
+ else:
+
+ print("something else happened")
+ #update and calculate our percentages and numbers for buffers/deletions
+ delNum.append(round(delCount,1))
+ bufNum.append(round(bufCount,1))
+ #print(bufCount,len(frameList))
+ bufPercent = (bufCount/len(frameList))*100
+ delPercent = (delCount/len(frameList))*100
+ bufPercentList.append(round(bufPercent,2))
+ delPercentList.append(round(delPercent,2))
+
+ #print(delNum,bufNum,bufPercentList,delPercentList)
+ #print("deleted frames:",delCount)
+ #print("buffered frames:",bufCount)
+ n +=1
+
+ #create our data frame for both times and frames
+ frameTable = pd.DataFrame(frameList)
+ columnNames = ['Master Timeline'] + camNames
+ frameTable.columns = columnNames
+ timeTable = pd.DataFrame(timeList)
+ timeTable.columns = columnNames
+ totalFrameRate = [round(1/intvl,1) for intvl in totalFrameIntvl]
+ frameRate = 1/totalAverageIntvl #calculates our framerate
+ results = {'Cam':camNames,'#Del':delNum,'%Del':delPercentList,'#Buf':bufNum,'%Buf':bufPercentList,'FPS':totalFrameRate}
+ resultTable = pd.DataFrame(results,columns = ['Cam','#Del','%Del','#Buf','%Buf','FPS'])
+
+ #============================================== plot data
+ differenceFrame = newFrame.diff(axis = 0)
+ fpsFrame = differenceFrame.pow(-1)
+ fig = Figure(figsize = [10,10])
+ fig.patch.set_facecolor('#F0F0F0')
+ a = fig.add_subplot(221)
+ a.set_xlabel('Frame')
+ a.set_ylabel('Time(s)')
+ newFrame.plot(ax = a, title = 'Camera timestamps')
+ b = fig.add_subplot(222)
+ b.set_xlabel('Frame')
+ b.set_ylabel('Frame duration (s)')
+ differenceFrame.plot(ax = b, title = 'Camera frame duration')
+ c = fig.add_subplot(223)
+ c.set_xlabel('Time (s)')
+ differenceFrame.plot.hist(ax = c, bins=6,alpha = .5, xlim = [0,.1], title = 'Frame interval distribution (s)')
+ d = fig.add_subplot(224)
+ d.set_xlabel('Frames per second')
+ fpsFrame.plot.hist(ax = d, bins=6,alpha = .5, xlim = [0,40], title = 'Frames per second distribution')
+ root = Tk()
+
+ proceedGUI(root,resultTable,fig)
+ root.mainloop()
\ No newline at end of file
diff --git a/freemocap/webcamrecord.py b/freemocap/webcamrecord.py
new file mode 100644
index 00000000..bd1c3dba
--- /dev/null
+++ b/freemocap/webcamrecord.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Dec 7 14:16:45 2020
+
+@author: Rontc
+"""
+
+
+from freemocap import recordingconfig
+from freemocap.webcam import recordGUI,camsetup,runrecordingprocess
+
+import tkinter as tk
+from pathlib import Path
+import datetime
+import os
+from ruamel.yaml import YAML
+
+def recordWebcamVideos(session):
+ #%% -----------------------------------------------SESSION INFO
+ #Choose a file path
+ userPath = '' #add custom path here if desired as r'[filepath]\', and for right now **you should have two backslashes at the end of your path**
+ if not userPath:
+ filepath = Path.cwd()
+ else:
+ filepath = userPath
+
+ #create session ID
+ sessionID_in = datetime.datetime.now().strftime("sesh_%y-%m-%d_%H%M%S")
+ session.sessionID = sessionID_in
+
+ #load the parameters YAML and extract the saved parameters if possible, and the default otherwise
+ parameter_path = filepath/'parameters.yaml'
+ parameters_yaml = YAML()
+ parameters = parameters_yaml.load(parameter_path)
+
+ try:
+ rotation_entry = parameters['saved']['rotations']
+ parameter_entry = parameters['saved']['parameters']
+ except:
+ print("Using Default Config")
+ rotation_entry = parameters['default']['rotations']
+ parameter_entry = parameters['default']['parameters']
+ #---------------------------------------------Run With GUI
+
+ #run the GUI to get the tasks, the cams chosen, the camera settings, and the session ID
+ task, cam_inputs, rotDict, paramDict, session.sessionID = recordGUI.RunGUI(sessionID_in,rotation_entry,parameter_entry)
+
+
+ #update the saved parameters in the YAML
+ parameters['saved']['rotations'] = rotDict
+ parameters['saved']['parameters'] = paramDict
+ parameters_yaml.dump(parameters, parameter_path)
+
+ #create a list from the rotation dictionary to be used in running webcams
+ rotation_input = list(rotDict.values())
+
+ proceedToCalibration = ''
+ # %% TASK CHOICES
+
+ #-----------------------------------------------SETUP
+ #when testing, press 'escape' to individually exit each feed. Camera input number associated with feed is displayed up top
+
+ if task == 'setup': #don't change this boolean by accident pls
+
+ if not cam_inputs:
+ raise ValueError('Camera input list (cam_inputs) is empty')
+
+ camsetup.RunSetup(cam_inputs,rotation_input,paramDict)
+ continueToRecording,session.sessionID = recordGUI.RunProceedtoRecordGUI(sessionID_in)
+
+ #proceedToCalibration = ''
+
+ if continueToRecording:
+ config_yaml_path = recordingconfig.createSession(session,filepath)
+ recordingconfig.createSessionTxt(session,paramDict,rotDict)
+ proceedToCalibration = runrecordingprocess.RunCams(session,cam_inputs,paramDict,rotation_input) #press ESCAPE to end the recording
+
+ #-----------------------------------------------RECORD
+ #Press ESCAPE to stop the recording process, and continue onto the time-syncing/editing process
+
+ elif task == 'record':#don't change this boolean by accident pls
+ #if continueToRecording:
+ #create a recording session
+ config_yaml_path = recordingconfig.createSession(session,filepath)
+ recordingconfig.createSessionTxt(session,paramDict,rotDict)
+
+ #check that cams were chosen (these lines can be removed when we make it mandatory to use the GUI)
+ if not cam_inputs:
+ raise ValueError('Camera input list (cam_inputs) is empty')
+
+ #run the recording webcams and get back a table of results
+ proceedToCalibration = runrecordingprocess.RunCams(session,cam_inputs,paramDict,rotation_input) #press ESCAPE to end the recording
+
+
+ elif not task:
+ print('No task detected')
+
+
+ elif task == 'debug': #out-of-order for right now
+ #to debug, manually enter a sessionID and change the task, and the results table/figures from that session will be generated
+ recordPath = filepath/'Data'/sessionID
+ webcam.DebugTime(sessionID,recordPath)
+
+
+ return proceedToCalibration
+
+#if __name__ == "__main__":
+# runCams()
\ No newline at end of file
diff --git a/parameters.yaml b/parameters.yaml
new file mode 100644
index 00000000..e6236323
--- /dev/null
+++ b/parameters.yaml
@@ -0,0 +1,24 @@
+#default parameters
+
+
+default:
+ rotations: {}
+ parameters:
+ exposure: -5
+ resWidth: 640
+ resHeight: 480
+ framerate: 25
+ codec: DIVX
+
+#saved parameters
+
+
+
+saved:
+ rotations:
+ parameters:
+ exposure:
+ resWidth:
+ resHeight:
+ framerate:
+ codec:
diff --git a/runme_fmc_staged.py b/runme_fmc_staged.py
new file mode 100644
index 00000000..95610276
--- /dev/null
+++ b/runme_fmc_staged.py
@@ -0,0 +1,84 @@
+from freemocap import createvideo, initialization, runcams, calibrate, runopenpose, reconstruct3D, playskeleton, session
+
+
+from aniposelib.boards import CharucoBoard, Checkerboard
+
+import numpy as np
+
+
+# %% Inputs to edit
+
+stage = 8 #set your starting stage here (stage = 1 will run the pipeline from webcams)
+
+board = CharucoBoard(7, 5,
+ #square_length=1, # here, in mm but any unit works (JSM NOTE - just using '1' so resulting units will be in 'charuco squarelenghts`)
+ #marker_length=.8,
+ square_length = 7,
+ marker_length = 6.8,
+ marker_bits=4, dict_size=250)
+
+
+sesh = session.Session(useOpenPose=True,useDLC=False)
+sesh.input_stage = stage
+sesh.debug = True
+sesh.sessionID = 'sesh_21-05-28_113930' #fill in if you are running from Stage 2 onwards
+
+
+# %% Initialization
+
+initialization.initialize(sesh,stage,board)
+
+# %% Stage One
+if not stage > 1:
+ runcams.RecordCams(sesh, sesh.cam_inputs, sesh.parameterDictionary, sesh.rotationInputs)
+else:
+ print('Skipping Video Recording')
+
+
+# %% Stage Two
+if not stage > 2:
+ runcams.SyncCams(sesh, sesh.timeStampData,sesh.numCamRange,sesh.vidNames,sesh.camIDs)
+else:
+ print('Skipping Video Syncing')
+
+if not stage > 3:
+ sesh.cgroup, sesh.mean_charuco_fr_mar_dim = calibrate.CalibrateCaptureVolume(sesh,board)
+else:
+ print('Skipping Calibration')
+
+if not stage > 4:
+ runopenpose.runOpenPose(sesh)
+else:
+ print('Skipping Running OpenPose')
+
+if not stage > 5:
+ sesh.openPoseData_nCams_nFrames_nImgPts_XY = runopenpose.parseOpenPose(sesh)
+else:
+ print('Skipping Parse OpenPose')
+
+
+
+if not stage > 6:
+ openPose_params = sesh.openPoseData_nCams_nFrames_nImgPts_XY.shape[0:3]
+ sesh.skel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XY, openPose_params)
+
+ path_to_skel_points = sesh.dataArrayPath/'skeleton_points.npy'
+ np.save(path_to_skel_points, sesh.skel_fr_mar_dim)
+else:
+ print('Skipping Skeleton Reconstruction')
+
+
+if not stage > 7:
+ playskeleton.ReplaySkeleton(sesh,1,40,-90,-75)
+else:
+ print('Skipping Skeleton Plotting')
+
+
+if not stage > 8:
+ createvideo.createVideo(sesh)
+else:
+ print('Skipping Video Creation')
+
+
+
+f = 2
\ No newline at end of file