Free Motion Capture for Everyone πβ¨
TypeScript
Staged FMC pipeline
Aaron Cheriand6af8a58361f
diff --git a/freemocap/__init__.py b/freemocap/__init__.pynew file mode 100644index 00000000..ef4b4f30--- /dev/null+++ b/freemocap/__init__.py@@ -0,0 +1,7 @@+# -*- coding: utf-8 -*-+"""+Created on Mon Mar 1 13:39:34 2021++@author: Rontc+"""+diff --git a/freemocap/calibrate.py b/freemocap/calibrate.pynew file mode 100644index 00000000..eae7dfc2--- /dev/null+++ b/freemocap/calibrate.py@@ -0,0 +1,200 @@+from aniposelib.cameras import CameraGroup+from freemocap import stolenfromanipose++from aniposelib.utils import load_pose2d_fnames+import numpy as np+import cv2++import matplotlib.pyplot as plt+from mpl_toolkits.mplot3d import Axes3D+from freemocap import reconstruct3D, stolenfromanipose+import os+from rich.progress import track++++def CalibrateCaptureVolume(session,board):++ calVideoFrameLength = 60+ createCalibrationVideos(session,calVideoFrameLength)++ vidnames = []+ cam_names = []+ for count,thisVidPath in enumerate(session.calVidPath.glob('*.mp4'),start=1):+ vidnames.append([str(thisVidPath)])+ cam_names.append(str(count))+ session.numCams = count++ cgroup = stolenfromanipose.CameraGroup.from_names(cam_names, fisheye=True) #Looking through their code... it looks lke the 'fisheye=True' doesn't do much (see 2020-03-29 obsidian note)++++ calibrationFile = '{}_calibration.yaml'.format(session.sessionID)++ session.cameraCalFilePath = session.sessionPath/calibrationFile++ forceRunCameraExtrinsicsCalibration = True #set this to True to re-run the camera calibration and saveover the existing config file++ # if not session.cameraCalFilePath.exists() or forceRunCameraExtrinsicsCalibration: #if that config file doesn't exist, run the camera calibration whosists. If it does, then just load the toml+ ## this will take a few minutes+ ## it will detect the charuco board in the videos,+ ## then calibrate the cameras based on the detections, using iterative bundle adjustment+ # error,all_rows,merged = cgroup.calibrate_videos(vidnames, board) #JSM NOTE - in its original form, this method doesn't throw an error when a video fails to load. This is an easy fix!++ #%% This next bit is a copy-paste of the inner bits of `cgroup.calibrate_videos` - Part 1 of 2+ """Takes as input a list of list of video filenames, one list of each camera.+ Also takes a board which specifies what should be detected in the videos"""++ all_rows = cgroup.get_rows_videos(vidnames, board, verbose=True)++ cgroup.set_camera_sizes_videos(vidnames)++ if session.debug:+ fig = plt.figure(47290)+ for camNum in range(len(all_rows)):+ ax = fig.add_subplot(2,2,camNum+1)+ # cap = cv2.VideoCapture(vidnames[camNum][0]) #this is crazy inefficient - we should save out the first image somewhere and save it in an accessible location+ # ret, frame = cap.read()+ # plt.imshow(frame)+ # cap.release()+ for frNum in range(len(all_rows[camNum])):+ corners = np.squeeze(all_rows[camNum][frNum]['filled'])+ plt.plot(corners[:,0], corners[:,1],'.-')+ # xmin = np.nanmin(corners[:,0])*.9+ # xmax = np.nanmax(corners[:,0])*1.1+ # ymin = np.nanmin(corners[:,1])*.9+ # ymax = np.nanmax(corners[:,1])*1.1+ # ax.set_xlim(xmin, xmax)+ # ax.set_xlim(ymin, ymax)+ plt.show()++++#%% This next bit is a copy-paste of the inner bits of `cgroup.calibrate_videos` - Part 2 of 2+ #error, merged = cgroup.calibrate_rows(all_rows, board,+ # init_intrinsics=True,+ # init_extrinsics=True)+#%% End of the copy-paste of the inner bits of `cgroup.calibrate_videos` - END++ #JSM NOTE - need add method to extract Charuco board points from the calibrate_videos pipeline++ ## if you need to save and load+ ## example saving and loading for later+ error,merged = cgroup.calibrate_videos(vidnames, board)+ cgroup.dump(session.cameraCalFilePath) #JSM NOTE - let's just use .yaml's unless there is some reason to use .toml+ mergename = session.sessionPath/'merged.npy'+ np.save(mergename,merged)+ else:+ cgroup = CameraGroup.load(session.cameraCalFilePath) #load previous calibration config++ session.cgroup = cgroup+ n_frames= 40+ startframe = 0+ n_trackedPoints = 24+ framelist = range(startframe,startframe+n_frames)++ charucoarray = np.empty([session.numCams,n_frames,n_trackedPoints,1,2])+ charucoarray[:] = np.nan++ data = np.load(session.sessionPath/'merged.npy',allow_pickle = True)++ for cam in range(session.numCams):+ for count,frame in enumerate(framelist):+ try:+ charucoarray[cam][count] = data[frame][cam]['filled']+ except:+ print('failed frame:', frame)+ continue+ charuco_nCams_nFrames_nImgPts_XY = np.array(charucoarray)++ session.charuco_nCams_nFrames_nImgPts_XY = charuco_nCams_nFrames_nImgPts_XY++ charuco_params = charuco_nCams_nFrames_nImgPts_XY.shape[0:3]++ charuco_fr_mar_dim = reconstruct3D.reconstruct3D(session,charuco_nCams_nFrames_nImgPts_XY,charuco_params)++ mean_charuco_fr_mar_dim = np.nanmean(charuco_fr_mar_dim,axis = 0)++ # charry_reshaped = charucoarray.reshape(session.numCams, -1, 2)++ # char_flat = cgroup.triangulate(charry_reshaped, progress=True)+ # charReprojerr_flat = cgroup.reprojection_error(char_flat, charry_reshaped, mean=True)++ # char_fr_mar_dim = char_flat.reshape(n_frames, n_trackedPoints, 3)+ # charReprojErr_fr_mar_err = charReprojerr_flat.reshape(n_frames, n_trackedPoints)+++ if session.debug:+ fig = plt.figure()+ #mean charuco position+ ax1 = fig.add_subplot(111,projection='3d')+ ax1.cla()+ x = mean_charuco_fr_mar_dim[:][:,0]+ y = mean_charuco_fr_mar_dim[:][:,1]+ z = mean_charuco_fr_mar_dim[:][:,2]+ mx = np.nanmean(x)+ my = np.nanmean(y)+ mz = np.nanmean(z)+ ax1.scatter(x,y,z, marker='o')+ ax1.set_title('Mean Charuco Point Positions')++ ax1.set_xlabel('x')+ ax1.set_ylabel('y')+ ax1.set_zlabel('z')++ for camNum in range(len(cgroup.cameras)):+ np.append(mx, cgroup.cameras[camNum].tvec[0])+ np.append(my, cgroup.cameras[camNum].tvec[1])+ np.append(mz, cgroup.cameras[camNum].tvec[2])++ ax1.scatter(cgroup.cameras[camNum].tvec[0], cgroup.cameras[camNum].tvec[1], cgroup.cameras[camNum].tvec[2], marker = 'p')+++ mx = np.nanmean(mx)+ my = np.nanmean(my)+ mz = np.nanmean(mz)++ ax1.set_xlim(mx-100,mx+100)+ ax1.set_ylim(my-100,my+100)+ ax1.set_zlim(mz-100,mz+100)++ # #charuco points over time+ # ax2 = fig.add_subplot(122)+ # ax2.cla()+ # numPts = charuco_fr_mar_dim.shape[1]+ # for pp in range(numPts):+ # ax2.plot(charuco_fr_mar_dim[:,pp,:])+ # ax2.set_title('Charuco Point positions over time')+ plt.show()++ path_to_charuco_array = session.dataArrayPath/'charuco_points.npy'+ np.save(path_to_charuco_array, mean_charuco_fr_mar_dim)+ return cgroup,mean_charuco_fr_mar_dim++def createCalibrationVideos(session,calVideoFrameLength):+ vidList = os.listdir(session.syncedVidPath)+ framelist = list(range(calVideoFrameLength))+ codec = 'DIVX'+ for count,vid in enumerate(vidList,start=1):+ cam_name = "Cam{}".format(count)+ cap = cv2.VideoCapture(str(session.syncedVidPath/vid))+ fourcc = cv2.VideoWriter_fourcc(*codec)++ #grab resolution parameters from the videos+ resWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))+ resHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))+ framerate = int(cap.get(cv2.CAP_PROP_FPS))++ saveName = session.sessionID +'_trimmed_' + cam_name + '.mp4' #create a name for the trimmed video+ saveCalVidPath = str(session.calVidPath/saveName) #create an output path for the function++ success, image = cap.read() #start reading frames++ out = cv2.VideoWriter(saveCalVidPath, fourcc, framerate, (resWidth,resHeight))+ print('Trimming ' + cam_name)+ for frame in track(framelist):+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need+ success, image = cap.read()+ out.write(image)+ cap.release()+ out.release()\ No newline at end of filediff --git a/freemocap/createvideo.py b/freemocap/createvideo.pynew file mode 100644index 00000000..3ae52d2a--- /dev/null+++ b/freemocap/createvideo.py@@ -0,0 +1,28 @@+import glob+import cv2+from pathlib import Path+import os+++def createVideo(session):++ vidSavePath = str(session.sessionPath/'{}_outVid.mp4'.format(session.sessionID))+ fps = 30+ shape = 1078,647+ frame_array = []++ for filename in session.imOutPath.glob('*.png'):+ filename= str(filename)++ #reading each files+ img = cv2.imread(filename)+ #resized = cv2.resize(img,shape)+ #shape = img.shape[:2]+ resized = cv2.resize(img,shape)+ #inserting the frames into an image array+ frame_array.append(resized)+ out = cv2.VideoWriter(vidSavePath,cv2.VideoWriter_fourcc(*'DIVX'), fps, shape)+ for i in range(len(frame_array)):+ # writing to a image array+ out.write(frame_array[i])+ out.release()\ No newline at end of filediff --git a/freemocap/dlc.py b/freemocap/dlc.pynew file mode 100644index 00000000..bd002983--- /dev/null+++ b/freemocap/dlc.py@@ -0,0 +1,70 @@+# -*- coding: utf-8 -*-+"""+Created on Fri Feb 26 12:42:08 2021++@author: Rontc+"""+import glob+import numpy as np+import pandas as pd+import matplotlib.pyplot as plt+import matplotlib as mpl+++def ParseDLCdata(sesh): #note this is all sloppy af just to get a completed video together. Latter iterations will use DLC methods and be smarter about Pandas Dataframs and whatnot+++ dlcCSVPaths = sesh.dlcDataPath.glob('*.csv') #NOTE - Super hacky here. Need to fix these methods (replace with DLC native functions?)++ numCams = int(sesh.numCams)+ numFrames = sesh.numFrames+1++ if sesh.sessionID == 'test6_01_21a': #NOTE - THIS IS DUMB AND BAD+ numCols = 8 #this is the number of tracked points in DLC - hardcoding for now because I am a bad person+ else:+ numCols = 3 #NOTE - AGAIN< THIS IS DUMB AND BAD >++ dlc_nCams_nFrames_nImgPts_XYC = np.ndarray([numCams, numFrames, numCols, 3])+ dlc_nCams_nFrames_nImgPts_XYC.fill(np.nan)+ camNum = -1+ for thisCSVpath in dlcCSVPaths: #triple nested for loop so I don't have to figure out np.reshape lol+ camNum += 1+ thisCam_dlcDataFrame = pd.read_csv(thisCSVpath, skiprows=1, header = [0,1]) #NOTE - This is dumb and bad. No need for this pandas dataframe, I think+ thisCam_dlcNumpy = thisCam_dlcDataFrame.to_numpy()+ thisCam_dlcNumpy = thisCam_dlcNumpy[:, 1:] #remove first column (frame numbers)++ for thisFrame in range(thisCam_dlcNumpy.shape[0]):+ for thisImgPt in range(0,thisCam_dlcNumpy.shape[1],3):+ dlc_nCams_nFrames_nImgPts_XYC[camNum,thisFrame,int(thisImgPt/3), 0] = thisCam_dlcNumpy[thisFrame, thisImgPt]+ dlc_nCams_nFrames_nImgPts_XYC[camNum,thisFrame,int(thisImgPt/3), 1] = thisCam_dlcNumpy[thisFrame, thisImgPt+1]+ dlc_nCams_nFrames_nImgPts_XYC[camNum,thisFrame,int(thisImgPt/3), 2] = thisCam_dlcNumpy[thisFrame, thisImgPt+2]+++ if sesh.debug:+ fig = plt.figure()++ ax1 = fig.add_subplot(221)+ ax2 = fig.add_subplot(222)+ ax3 = fig.add_subplot(223)+ ax4 = fig.add_subplot(224)++ cam1im = sesh.firstImage_nCams_list[0]+ cam2im = sesh.firstImage_nCams_list[1]+ cam3im = sesh.firstImage_nCams_list[2]+ cam4im = sesh.firstImage_nCams_list[3]++ ax1.imshow(cam1im)+ ax1.plot(dlc_nCams_nFrames_nImgPts_XYC[0,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[0,0,:,1])++ ax2.imshow(cam2im)+ ax2.plot(dlc_nCams_nFrames_nImgPts_XYC[1,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[1,0,:,1])++ ax3.imshow(cam3im)+ ax3.plot(dlc_nCams_nFrames_nImgPts_XYC[2,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[2,0,:,1])++ ax4.imshow(cam4im)+ ax4.plot(dlc_nCams_nFrames_nImgPts_XYC[3,0,:,0],dlc_nCams_nFrames_nImgPts_XYC[3,0,:,1])++ plt.show()++ sesh.dlc_nCams_nFrames_nImgPts_XYC = dlc_nCams_nFrames_nImgPts_XYC\ No newline at end of filediff --git a/freemocap/initialization.py b/freemocap/initialization.pynew file mode 100644index 00000000..b787b427--- /dev/null+++ b/freemocap/initialization.py@@ -0,0 +1,180 @@++from pathlib import Path+from ruamel.yaml import YAML+import datetime+import sys+import pandas as pd+import numpy as np++from freemocap.webcam import recordGUI,camsetup+from freemocap import recordingconfig, stolenfromanipose++def initialize(session, stage, board):++ print('Starting initialization for stage {}'.format(stage))+ #if stage == 1:+ filepath = Path.cwd()+# %% Stage One Initialization+ if stage == 1:+++ #create session ID+ sessionID_in = datetime.datetime.now().strftime("sesh_%y-%m-%d_%H%M%S")+ session.sessionID = sessionID_in++ #load the parameters YAML and extract the saved parameters if possible, and the default otherwise+ parameter_path = filepath/'parameters.yaml'+ parameters_yaml = YAML()+ parameters = parameters_yaml.load(parameter_path)++ try:+ rotation_entry = parameters['saved']['rotations']+ parameter_entry = parameters['saved']['parameters']+ except:+ print("Using Default Config")+ rotation_entry = parameters['default']['rotations']+ parameter_entry = parameters['default']['parameters']++ proceedToRecording = False #create this boolean, set it to false, and if the user wants to record+ #later in the pipeline, it will be set to true++ #run the GUI to get the tasks, the cams chosen, the camera settings, and the session ID+ task, cam_inputs, rotDict, paramDict, session.sessionID = recordGUI.RunGUI(sessionID_in,rotation_entry,parameter_entry)++ #update the saved parameters in the YAML+ parameters['saved']['rotations'] = rotDict+ parameters['saved']['parameters'] = paramDict+ parameters_yaml.dump(parameters, parameter_path)++ #create a list from the rotation dictionary to be used in running webcams+ rotation_input = list(rotDict.values())++ if task == 'setup':+ #run setup processes, and then check if th user wants to proceed to recording+ camsetup.RunSetup(cam_inputs,rotation_input,paramDict)+ proceedToRecording,session.sessionID = recordGUI.RunProceedtoRecordGUI(sessionID_in)++ elif task == 'record':+ proceedToRecording = True++ if proceedToRecording:+ #create these session properties to be used later in the pipeline+ session.cam_inputs = cam_inputs+ session.parameterDictionary = paramDict+ session.rotationInputs = rotation_input++ #create a config yaml and text file for this session+ config_yaml_path = recordingconfig.createSession(session,filepath)+ recordingconfig.createSessionTxt(session,paramDict,rotDict)++ print('Proceeding to Stage One')++ else:++ sys.exit('Recording Cancelled')+# %% Stage Two Initalization+ if stage == 2:+ session.sessionPath = filepath/'Data'/session.sessionID #create a session path based on the sessionID++ #load the config yaml for this session, and add all the paths to the session file+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID)+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class++ #from the config settings add the camera input parameters to parameter dictionary+ session.parameterDictionary = config_settings['CamInputs']['ParameterDict']+ session.rotationInputs = config_settings['CamInputs']['RotationInputs']++ #initialize the path to the timestamp csv+ csvName = session.sessionID + '.csv'+ csvPath = session.sessionPath/csvName++ #read CSV data, turn it into a data frame+ timeStampData = pd.read_csv (csvPath)+ timeStampData = timeStampData.iloc[:,1:]++ #get the camIDs and number of cameras (numCamRange) from the dataframe+ camIDs = list(timeStampData.columns)+ numCamRange = range(len(camIDs))++ #create names for each of the raw videos+ vidNames = []+ for x in numCamRange:+ singleVidName = 'raw_cam{}.mp4'.format(x+1)+ vidNames.append(singleVidName)+++ #initialize all the session variables we'll need to run the rest of the pipeline+ session.timeStampData = timeStampData+ session.camIDs = camIDs+ session.numCamRange = numCamRange+ session.vidNames = vidNames++ if stage == 3:++ session.sessionPath = filepath/'Data'/session.sessionID #create a session path based on the sessionID++ #load the config yaml for this session, and add all the paths to the session file+ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID)++ if session.yamlPath.is_file(): #if the config yaml exists (from a webcam recording)+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class+ else: #if it doesn't exist (because of a GoPro/external camera recording)+ recordingconfig.createSession(session,filepath)++ if stage == 4:++ session.sessionPath = filepath/'Data'/session.sessionID++ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class++ if stage == 5:++ session.sessionPath = filepath/'Data'/session.sessionID++ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class++ session.openPose_jsonPathList = config_settings['openPose_jsonPathList']+ session.numCams = len(session.openPose_jsonPathList)++ if stage == 6:++ session.sessionPath = filepath/'Data'/session.sessionID++ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class++ session.openPoseData_nCams_nFrames_nImgPts_XY = np.load(session.dataArrayPath/'openPoseData_nCams_nFrames_nImgPts_XY.npy')+ calibrationFile = '{}_calibration.yaml'.format(session.sessionID)+ session.cameraConfigFilePath = session.sessionPath/calibrationFile+ session.cgroup = stolenfromanipose.CameraGroup.load(session.cameraConfigFilePath)++ if stage == 7:++ session.sessionPath = filepath/'Data'/session.sessionID++ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class++ session.openPose_imgPathList = config_settings['openPose_imgPathList']+ session.numCams = len(session.openPose_imgPathList)++ session.mean_charuco_fr_mar_dim = np.load(session.dataArrayPath/'charuco_points.npy')+ session.skel_fr_mar_dim = np.load(session.dataArrayPath/'skeleton_points.npy')++ if stage == 8:++ session.sessionPath = filepath/'Data'/session.sessionID++ session.yamlPath = session.sessionPath/'{}_config.yaml'.format(session.sessionID) #path to the configuration yaml+ config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs+ recordingconfig.load_session_paths(session,config_settings) #add paths to session class++# %%diff --git a/freemocap/playskeleton.py b/freemocap/playskeleton.pynew file mode 100644index 00000000..757aef3b--- /dev/null+++ b/freemocap/playskeleton.py@@ -0,0 +1,256 @@+# -*- coding: utf-8 -*-+"""+Created on Fri Feb 26 12:51:21 2021++@author: Rontc+"""++import numpy as np+#from mpl_toolkits.mplot3d import Axes3D+import matplotlib.pyplot as plt+import matplotlib as mpl+import cv2+from pathlib import Path+#from glob import glob++# def PlaySkeleton(session, vidType):++# # vidType = 0 #No Cam Ims+# # vidType = 1 #Only Cam1 im+# # vidType = 2 #All Cam Ims++# #where to put the output images (later will be justa video)+++# #os.chdir(imOutPath)+++++# #set axis limits based on charuco board+# # mx = np.mean(charuco_x)+# # my = np.mean(charuco_y)+# # mz = np.mean(charuco_z)++# #set up figure+# # fig = plt.figure(figsize=([11,8]))++# # if vidType==0:+# # axMain = fig.add_subplot(111,projection='3d')+# # elif vidType == 1:+# # axMain = fig.add_subplot(121,projection='3d')+# # axCam1 = fig.add_subplot(122)+# # elif vidType == 2:+# # figGridSpec = fig.add_gridspec(2,4)+# # axMain = fig.add_subplot(figGridSpec[:2,:2],projection='3d')+# # axCam1 = fig.add_subplot(figGridSpec[0,2])+# # axCam2 = fig.add_subplot(figGridSpec[0,3])+# # axCam3 = fig.add_subplot(figGridSpec[1,2])+# # axCam4 = fig.add_subplot(figGridSpec[1,3])+++++# ##############################################################################################+# ##Frame-by-frame animation loop starts here+# ##############################################################################################++# # fig = plt.figure()+# # ax = fig.add_subplot(111,projection='3d')+# # ax.cla()+# # x = session.mean_charuco_fr_mar_dim[:][:,0]+# # y = session.mean_charuco_fr_mar_dim[:][:,1]+# # z = session.mean_charuco_fr_mar_dim[:][:,2]+# # ax.scatter(x,y,z, marker='o')+# # plt.show()++# # define Skeleton connections+# # head = [17, 15, 0, 16, 18]+# head = [ 15, 0, 16]+# spine = [0,1,8]+# rArm = [17, 15, 0, 16, 18]+# rArm = [4 ,3 ,2 ,1]+# lArm = [1, 5, 6, 7]+# rLeg = [11 ,10, 9, 8]+# lLeg = [14 ,13 ,12, 8]+# rFoot = [11, 23,22, 11, 24]+# lFoot = [14, 20, 19, 14, 21]++# #Make some handy maps ;D+# rHandIDstart = 25+# lHandIDstart = rHandIDstart+21++# thumb = np.array([0,1,2,3,4])+# index = np.array([0, 5,6,7,8])+# bird = np.array([0, 9,10,11,12])+# ring = np.array([0, 13,14,15,16])+# pinky = np.array([0, 17,18,19,20])+++# fig = plt.figure(figsize=(8,6))+# axMain = fig.add_subplot(111,projection='3d')+# plt.ion()+# for fr in range(session.numFrames):+# fig.suptitle(['Frame# - ', str(fr)])+# axMain.cla()+# char_x = session.mean_charuco_fr_mar_dim[:][:,0]+# char_y = session.mean_charuco_fr_mar_dim[:][:,1]+# char_z = session.mean_charuco_fr_mar_dim[:][:,2]++# if session.useOpenPose:+# sk_x = session.skel_fr_mar_dim[fr,:,0] #skeleton x data+# sk_y = session.skel_fr_mar_dim[fr,:,1] #skeleton y data+# sk_z = session.skel_fr_mar_dim[fr,:,2] #skeleton z data++# axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)+# axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)+# axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)+# axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)+# axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)+# axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)+# axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)+# axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)+# axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)++# # plot handybois+# # right hand+# axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+# axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+# axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+# axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+# axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)++# #left hand+# axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+# axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+# axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+# axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+# axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)++# axMain.scatter(char_x,char_y,char_z, marker='o')++# axMain.set_xbound(lower=-150, upper=150)+# axMain.set_ybound(lower=-150, upper=150)+# axMain.set_zbound(lower=-150, upper=150)++# #imName = [session.sessionID + '_frame' + fr.zfill(6)]+# frameName = str(fr).zfill(6)+# saveFrameName = '{}.png'.format(frameName)+# saveFramePath = session.imOutPath/saveFrameName+# fig.savefig(saveFramePath)# , bbox_inches = 'tight')++# plt.pause(0.01)+# plt.show()+++def ReplaySkeleton(session, vidType, startframe,azimuth,elevation):+ camImgPathList = {}+ for cam in range(session.numCams):+ camImgPathList[cam] = list(sorted(Path(session.openPose_imgPathList[cam]).glob('*.png')))+ session.numFrames = len(camImgPathList[cam]) #will need to perhaps put a check in on whether numFrames between cameras are the same+++# define Skeleton connections+# head = [17, 15, 0, 16, 18]+ head = [ 15, 0, 16]+ spine = [0,1,8]+ rArm = [17, 15, 0, 16, 18]+ rArm = [4 ,3 ,2 ,1]+ lArm = [1, 5, 6, 7]+ rLeg = [11 ,10, 9, 8]+ lLeg = [14 ,13 ,12, 8]+ rFoot = [11, 23,22, 11, 24]+ lFoot = [14, 20, 19, 14, 21]++ #Make some handy maps ;D+ rHandIDstart = 25+ lHandIDstart = rHandIDstart+21++ thumb = np.array([0,1,2,3,4])+ index = np.array([0, 5,6,7,8])+ bird = np.array([0, 9,10,11,12])+ ring = np.array([0, 13,14,15,16])+ pinky = np.array([0, 17,18,19,20])+++ fig = plt.figure(figsize=(11,8))++ if vidType == 0:+ axMain = fig.add_subplot(111,projection='3d')+ elif vidType == 1:+ axMain = fig.add_subplot(121,projection='3d')+ axCam1 = fig.add_subplot(122)++ # fig.tight_layout(h_pad = 5)+ axMain.view_init(azim = azimuth, elev= elevation)+ plt.ion()+ for fr in range(startframe,session.numFrames):+ fig.suptitle(['Frame# - ', str(fr)])+ axMain.cla()++ if vidType == 1:+ axCam1.cla()+++ char_x = session.mean_charuco_fr_mar_dim[:][:,0]+ char_y = session.mean_charuco_fr_mar_dim[:][:,1]+ char_z = session.mean_charuco_fr_mar_dim[:][:,2]++ if session.useOpenPose:+ sk_x = session.skel_fr_mar_dim[fr,:,0] #skeleton x data+ sk_y = session.skel_fr_mar_dim[fr,:,1] #skeleton y data+ sk_z = session.skel_fr_mar_dim[fr,:,2] #skeleton z data++ axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)+ axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)+ axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)+ axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)++ # plot handybois+ # right hand+ axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)++ #left hand+ axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)++ #plot charuco grid+ axMain.scatter(char_x,char_y,char_z, marker='o')++ # #plot camera positions (I think...)+ # for camNum in range(len(session.cgroup.cameras)):+ # axMain.scatter(session.cgroup.cameras[camNum].tvec[0], session.cgroup.cameras[camNum].tvec[1], session.cgroup.cameras[camNum].tvec[2], marker = 'p')++ axMain.set_xlabel('x')+ axMain.set_ylabel('y')+ axMain.set_zlabel('z')++ axMain.set_xlim(-100,100)+ axMain.set_ylim(-100,100)+ axMain.set_zlim(100,300)++ if vidType ==0:+ pass+ elif vidType == 1:+ axCam1.imshow(cv2.cvtColor(cv2.imread(str(camImgPathList[0][fr])),cv2.COLOR_BGR2RGB))+ axCam1.axis('off')++ plt.pause(0.01)+ plt.show()++ frameName = str(fr).zfill(6)+ saveFrameName = '{}.png'.format(frameName)+ saveFramePath = session.imOutPath/saveFrameName+ fig.savefig(saveFramePath )diff --git a/freemocap/reconstruct3D.py b/freemocap/reconstruct3D.pynew file mode 100644index 00000000..a1a79142--- /dev/null+++ b/freemocap/reconstruct3D.py@@ -0,0 +1,15 @@+def reconstruct3D(session, data_nCams_nFrames_nImgPts_XY,data_params):+ n_cams, n_frames, n_trackedPoints = data_params++ #assert numDims == 2, 'Problem with your data! Not enough dimensions'++ dataFlat_nCams_nTotalPoints_XY = data_nCams_nFrames_nImgPts_XY.reshape(n_cams, -1, 2) #reshape data to collapse across 'frames' so it becomes [numCams, numFrames*numPoints, XY]++ data3d_flat = session.cgroup.triangulate(dataFlat_nCams_nTotalPoints_XY, progress=True)+ dataReprojerr_flat = session.cgroup.reprojection_error(data3d_flat, dataFlat_nCams_nTotalPoints_XY, mean=True)++ ##return:+ data_fr_mar_dim = data3d_flat.reshape(n_frames, n_trackedPoints, 3)+ dataReprojErr_fr_mar_err = dataReprojerr_flat.reshape(n_frames, n_trackedPoints)++ return data_fr_mar_dimdiff --git a/freemocap/recordingconfig.py b/freemocap/recordingconfig.pynew file mode 100644index 00000000..4473559d--- /dev/null+++ b/freemocap/recordingconfig.py@@ -0,0 +1,132 @@+# -*- coding: utf-8 -*-+"""+Created on Thu Mar 11 11:54:12 2021++@author: Rontc+"""+from ruamel.yaml import YAML+from pathlib import Path++def createSession(session,filepath, existing_data = False):+ #create file paths for the session, raw and synced videos++ pathList = []++ recordPath = filepath/'Data'+ recordPath.mkdir(exist_ok= True)++ session.sessionPath = recordPath/session.sessionID+ #if existing_data:+ #session.sessionPath.mkdir(exist_ok = True)+ #else:+ session.sessionPath.mkdir(exist_ok= True)+ pathList.append(session.sessionPath)++ session.rawVidPath = session.sessionPath/'RawVideos'+ session.rawVidPath.mkdir(exist_ok = True)+ pathList.append(session.rawVidPath)++ session.syncedVidPath = session.sessionPath/'SyncedVideos'+ session.syncedVidPath.mkdir(exist_ok = True)+ pathList.append(session.syncedVidPath)++ session.calVidPath = session.sessionPath/'CalVideos'+ session.calVidPath.mkdir(exist_ok = True)+ pathList.append(session.calVidPath)++ session.openPoseDataPath = session.sessionPath/'OpenPoseData'+ session.openPoseDataPath.mkdir(exist_ok = True)+ pathList.append(session.openPoseDataPath)++ session.dlcDataPath = session.sessionPath / 'DLCdata' / 'videos'+ session.dlcDataPath.mkdir(exist_ok = True, parents = True)+ pathList.append(session.dlcDataPath)++ session.imOutPath = session.sessionPath / 'imOut'+ session.imOutPath.mkdir(exist_ok=True)+ pathList.append(session.imOutPath)++ session.dataArrayPath = session.sessionPath/'Data Arrays'+ session.dataArrayPath.mkdir(exist_ok=True)+ pathList.append(session.dataArrayPath)++ #create a list of paths that is passed to the recording script+ #pathList = [session.sessionPath,session.rawVidPath,session.syncedVidPath,session.openPoseDataPath,session.dlcDataPath,session.imOutPath, session.dataArrayPath]+++ config_settings,config_yaml = create_config_yaml()+ yaml_name = session.sessionPath/'{}_config.yaml'.format(session.sessionID)+ session.yamlPath = yaml_name+ # print(yaml_name)+ write_config_yaml(session,config_settings,config_yaml,pathList,yaml_name)++ return yaml_name++def createSessionTxt(session,paramDict,rotDict):+ #create a text file listing recording parameters+ parameter_text = session.sessionPath/'sessionSettings.txt'+ text = open(parameter_text, 'w')+ text.write("Session ID = %s\n" %(session.sessionID))+ text.write("%s = %s\n" %("Parameters", paramDict))+ text.write("%s = %s\n" %("Rotations", rotDict))+ text.close()+++++def create_config_yaml():+++ yaml_str = """\+ # Camera Inputs+ CamInputs:+ RotationInputs:+ ParameterDict:++ # Paths+ Paths:+ sessionPath:+ rawVidPath:+ syncedVidPath:+ calVidPath:+ openPoseDataPath:+ dlcDataPath:+ imOutPath:+ dataArrayPath:+ \n+ """+ config_yaml = YAML()+ config_settings = config_yaml.load(yaml_str)+ return config_settings,config_yaml+++def write_config_yaml(session,config_settings,config_yaml,pathList,yaml_name):++ for count,key in enumerate(config_settings['Paths'].keys()):+ config_settings['Paths'][key] = str(pathList[count])++ config_settings['CamInputs']['RotationInputs'] = session.rotationInputs+ config_settings['CamInputs']['ParameterDict'] = session.parameterDictionary+++ #yamlPath = sessionPath/'config_yaml'+ #print(config_settings,yaml_name)+ with open(yaml_name, 'w') as outfile:+ config_yaml.dump(config_settings, outfile)+++def load_config_yaml(yaml_path):+ config_yaml = YAML()+ with open(yaml_path,'r') as fp:+ config_settings = config_yaml.load(fp)+ return config_settings++def load_session_paths(session,config_settings):+ configPaths = config_settings['Paths']+ session.rawVidPath = Path(configPaths['rawVidPath'])+ session.syncedVidPath = Path(configPaths['syncedVidPath'])+ session.calVidPath = Path(configPaths['calVidPath'])+ session.openPoseDataPath = Path(configPaths['openPoseDataPath'])+ session.dlcDataPath = Path(configPaths['dlcDataPath'])+ session.imOutPath = Path(configPaths['imOutPath'])+ session.dataArrayPath = Path(configPaths['dataArrayPath'])diff --git a/freemocap/runcams.py b/freemocap/runcams.pynew file mode 100644index 00000000..96c2dee9--- /dev/null+++ b/freemocap/runcams.py@@ -0,0 +1,81 @@++from freemocap.webcam import startcamrecording, timesync, videotrim++from pathlib import Path+import time+import pickle+import pandas as pd+import numpy as np+from tkinter import Tk+++def RecordCams(session,camInputs,parameterDictionary,rotationInputs):++ #%% Setting up recordings+ beginTime = time.time()+ numCams = len(camInputs) #number of cameras+ numCamRange = range(numCams) #a range for the number of cameras that we have+ vidNames = []+ camIDs = []+ for x in numCamRange: #create names for each of the initial untrimmed videos+ singleCamID = 'Cam{}'.format(x+1)+ camIDs.append(singleCamID) #creates IDs for each camera based on the number of cameras entered+ singleVidName = 'raw_cam{}.mp4'.format(x+1)+ vidNames.append(singleVidName)++ #%% Starting the thread recordings for each camera+ threads = []+ for n in numCamRange: #starts recording video, opens threads for each camera+ camRecordings = startcamrecording.CamRecordingThread(camIDs[n],camInputs[n],vidNames[n],session.rawVidPath,beginTime,parameterDictionary)+ camRecordings.start()++ threads.append(camRecordings)++ for camRecordings in threads:+ camRecordings.join() #make sure that one thread ending doesn't immediately end all the others (before they can dump data in a pickle file)++ print('finished recordings')++ timeStampList = []++ for e in numCamRange: #open the saved pickle file for each camera, and add the timestamps to the dataList list+ with open(camIDs[e], 'rb') as f:+ camTimeList = pickle.load(f)+ timeStampList.append(camTimeList)++ timeDictionary = {}++ id_and_time = zip(camIDs,timeStampList)++ for cam,data in id_and_time:+ timeDictionary[cam] = np.array(data) #create a dictionary that holds the timestamps for each camera+++ df = pd.DataFrame.from_dict(timeDictionary, orient = 'index') #create a data frame from this dictionary+ timeStampData = df.transpose()+ csvName = session.sessionID + '.csv' #create our csv filename+ csvPath = session.sessionPath/csvName+ timeStampData.to_csv(csvPath) #turn dataframe into a CSV++ session.numCams = numCams+ session.timeStampData = timeStampData+ session.camIDs = camIDs+ session.numCamRange = numCamRange+ session.vidNames = vidNames++def SyncCams(session, timeStampData,numCamRange,vidNames,camIDs):++ #start the timesync process+ frameTable,timeTable,frameRate,resultsTable,plots = timesync.TimeSync(timeStampData,numCamRange,camIDs)++ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no+ root = Tk()+ proceed = timesync.proceedGUI(root,resultsTable,plots) #create a GUI instance called proceed+ root.mainloop()++ if proceed.proceed == True:+ print()+ print('Starting editing')+ videotrim.VideoTrim(session,vidNames,frameTable,session.parameterDictionary,session.rotationInputs,numCamRange)+ #videotrim.createCalibrationVideos(session,60,parameterDictionary)+ print('all done')\ No newline at end of filediff --git a/freemocap/runopenpose.py b/freemocap/runopenpose.pynew file mode 100644index 00000000..ab223ed3--- /dev/null+++ b/freemocap/runopenpose.py@@ -0,0 +1,134 @@+import os+import subprocess+import numpy as np+from rich.progress import track+from ruamel.yaml import YAML+from pathlib import Path++import glob+import json++def runOpenPose(session, dummyRun=False):++ if not dummyRun:+ os.chdir(session.openPoseExePath)++ openPose_jsonPathList = [] #list to hold the paths to the json files+ openPose_imgPathList = []+ openPose_imgPathList_yaml = []+ openPose_jsonPathList_yaml = []++ for thisVidPath in session.syncedVidPath.iterdir(): #Run OpenPose ('Windows Portable Demo') on each video in the raw video folder+ if thisVidPath.suffix =='.mp4': #NOTE - build some list of 'synced video names' and check against that+ print('OpenPosing: ', thisVidPath.name )+ vidPath = session.openPoseDataPath / thisVidPath.stem+ jsonPath = vidPath / 'json'+ jsonPath.mkdir(parents=True, exist_ok=True) #this camera's json files (with keypoints)+ imgPath = vidPath / 'img'+ imgPath.mkdir(parents=True, exist_ok=True)+ openPose_jsonPathList.append(jsonPath)+ openPose_imgPathList.append(imgPath)+ openPose_imgPathList_yaml.append(str(imgPath))+ openPose_jsonPathList_yaml.append(str(jsonPath))+ if not dummyRun:+ subprocess.run(['bin\OpenPoseDemo.exe', '--video', str(thisVidPath), '--write_json', str(jsonPath),'--write_images', str(imgPath),'--write_images_format','png','--net_resolution','-1x320','--hand','--face', '--number_people_max', '1'],shell = True)+ else:+ print('Skipping: ', thisVidPath.name )+ session.openPose_jsonPathList = openPose_jsonPathList+ session.openPose_imgPathList = openPose_imgPathList++ yaml = YAML()+ data = yaml.load(session.yamlPath)+ data['openPose_imgPathList'] = openPose_imgPathList_yaml+ data['openPose_jsonPathList'] = openPose_jsonPathList_yaml+ yaml.dump(data, session.yamlPath)++ os.chdir(session.sessionPath)+++def parseOpenPose(session):+ thisCamNum = -1++ ## %%+ #build header for dataframe - NOTE - #openpose data comes in a line ordered 'pixel x location (px)', 'pixel y (py)', 'confidence (conf)' for each keypoint++ dataFrameHeader = []+ bodyCols = 75+ handCols = 63 #per hand+ faceCols = 210 #das alotta face!+ headerLength = bodyCols + 2*handCols + faceCols#should be 411 for whatever version of openpose i was using on 11 Jan 2021+ numImgPoints = headerLength++ for bb in range(0,int(bodyCols/3)): #loop through the number of body markers (i.e. #bodyCols/3)+ dataFrameHeader.append('body_' + str(bb).zfill(3) + '_pixx')+ dataFrameHeader.append('body_' + str(bb).zfill(3) + '_pixy')+ dataFrameHeader.append('body_' + str(bb).zfill(3) + '_conf')++ for hr in range(0,int(handCols/3)): #loop through the number of handR markers (i.e. #handCols/3)+ dataFrameHeader.append('handR_' + str(hr).zfill(3) + '_pixx')+ dataFrameHeader.append('handR_' + str(hr).zfill(3) + '_pixy')+ dataFrameHeader.append('handR_' + str(hr).zfill(3) + '_conf')++ for hl in range(0,int(handCols/3)): #loop through the number of handL markers (i.e. #handCols/3)+ dataFrameHeader.append('handL_' + str(hl).zfill(3) + '_pixx')+ dataFrameHeader.append('handL_' + str(hl).zfill(3) + '_pixy')+ dataFrameHeader.append('handL_' + str(hl).zfill(3) + '_conf')++ for ff in range(0,int(faceCols/3)): #loop through the number of Face markers (i.e. #faceCols/3)+ dataFrameHeader.append('face_' + str(ff).zfill(3) + '_pixx')+ dataFrameHeader.append('face_' + str(ff).zfill(3) + '_pixy')+ dataFrameHeader.append('face_' + str(ff).zfill(3) + '_conf')++ assert len(dataFrameHeader) == headerLength, ['Header is the wrong length! Should be ' + str(headerLength) + ' but it is ' + str(len(dataFrameHeader)) + ' Check version of OpenPose?']++ ## %%+ ## load in data from json files+ numFrames = int(len(list(Path(session.openPose_jsonPathList[0]).glob('*')))) #lol+ numMarkers= int(int(len(dataFrameHeader)/3))+ numCams = int(session.numCams)++ openPoseData_nCams_nFrames_nImgPts_XYC = np.ndarray([numCams,numFrames,numMarkers,3]) #hardcoding for now because I am a bad person++ for thisCams_JsonFolderPath in track(session.openPose_jsonPathList, description='Parsing json\'s into a dataframe (per cam)' ):+ thisCamNum += 1+ # print('Parsing into a dataframe: ', thisCams_JsonFolderPath.name )+ jsonPaths = sorted(Path(thisCams_JsonFolderPath).glob('*.json')) #glob is a "generator(?)" for paths to all the jason for THIS camara++ for thisJsonPath in jsonPaths: #loop throug all the json files and save their 'people' data to a dictionary (which will then be formatted into a pandas dataframe). NOTE - will be empty array if no hoomans visible in frame+ # print('loading: ', thisJsonPath.name)+ frameNum = int(thisJsonPath.stem.split('_')[-2]) #frame number we're on+ thisJsonData = json.loads(thisJsonPath.read_bytes())+ thisJsonData = thisJsonData['people'] # #FEATURE_REQUEST - at some point, we should check the openpose version (save it with the data somehow, verify everything is the same version, use different markernamlists for different versions, etc)++ if thisJsonData: #if this json has data+ bodyData = np.array(thisJsonData[0]['pose_keypoints_2d'])+ handRData = np.array(thisJsonData[0]['hand_right_keypoints_2d'])+ handLData = np.array(thisJsonData[0]['hand_left_keypoints_2d'])+ faceData = np.array(thisJsonData[0]['face_keypoints_2d'])+ thisFrameRow = np.hstack((bodyData,handRData, handLData, faceData)) #horizontally concatenate these arrays+ else: #if this json is empty, just stuff it fulla NaNs+ thisFrameRow = np.empty([headerLength])+ thisFrameRow.fill(np.nan)+++ assert thisFrameRow.size == headerLength, ['Header is the wrong length! Should be ' + str(headerLength) + ' but it is ' + str(thisFrameRow.size) + ' Check version of OpenPose?']++ if frameNum < openPoseData_nCams_nFrames_nImgPts_XYC.shape[1]: #NOTE: THIS SHOULDN"T BE NECESSARY. CURRENT Vid Trim Methods can (apparently) produce vids with off-by-one numbers of frames+ openPoseData_nCams_nFrames_nImgPts_XYC[thisCamNum, frameNum, :, :] = np.reshape(thisFrameRow, [137,3]) #hard coding b/c I'm a bad person++++ openPoseData_nCams_nFrames_nImgPts_XY = openPoseData_nCams_nFrames_nImgPts_XYC[:,:,:,0:2].copy()+ openpose_confidence = openPoseData_nCams_nFrames_nImgPts_XYC[:,:,:,2].copy()++ openpose_score_threshold = .3+ openPoseData_nCams_nFrames_nImgPts_XY[openpose_confidence < openpose_score_threshold] = np.nan #replace low confidence points with '++ session.dataFrameHeader = dataFrameHeader+ session.numImgPoints = numImgPoints+ session.numFrames = frameNum #NOTE - Need to find a safer way to get this number++ path_to_openpose_2d = session.dataArrayPath/'openPoseData_nCams_nFrames_nImgPts_XY.npy'+ np.save(path_to_openpose_2d, openPoseData_nCams_nFrames_nImgPts_XY)++ return openPoseData_nCams_nFrames_nImgPts_XY\ No newline at end of filediff --git a/freemocap/session.py b/freemocap/session.pynew file mode 100644index 00000000..a0ab3501--- /dev/null+++ b/freemocap/session.py@@ -0,0 +1,16 @@+class Session: #session like "recording session"+ def __init__(self, useOpenPose, useDLC):+ self.sessionID = '' #The sessionID tag will be used to generate files names and whatnot+ self.sessionPath = '' #The folder where the to-be-processed videos live (in a folder called "synced Vids")+ self.numCams = ''#The number of cameras used in this recording session+ self.openPoseDataPath = ''#Where the open pose data lives+ self.dlcDataPath = ''#Where the DLC data lives+ self.useOpenPose = useOpenPose+ self.useDLC = useDLC+ self.openPoseExePath = r'C:\openpose'++ self.cam_inputs = None+ self.parameterDictionary = None+ self.rotationInputs = None++\ No newline at end of filediff --git a/freemocap/stolenfromanipose.py b/freemocap/stolenfromanipose.pynew file mode 100644index 00000000..d703bc40--- /dev/null+++ b/freemocap/stolenfromanipose.py@@ -0,0 +1,1658 @@+import cv2+import numpy as np+from copy import copy+from scipy.sparse import lil_matrix, dok_matrix+from scipy import optimize+from scipy import signal+from numba import jit+from collections import defaultdict, Counter+import toml+import itertools+from tqdm import trange+from pprint import pprint+import time++from aniposelib.boards import merge_rows, extract_points, \+ extract_rtvecs, get_video_params+from aniposelib.utils import get_initial_extrinsics, make_M, get_rtvec, \+ get_connections++@jit(nopython=True, parallel=True)+def triangulate_simple(points, camera_mats):+ num_cams = len(camera_mats)+ A = np.zeros((num_cams * 2, 4))+ for i in range(num_cams):+ x, y = points[i]+ mat = camera_mats[i]+ A[(i * 2):(i * 2 + 1)] = x * mat[2] - mat[0]+ A[(i * 2 + 1):(i * 2 + 2)] = y * mat[2] - mat[1]+ u, s, vh = np.linalg.svd(A, full_matrices=True)+ p3d = vh[-1]+ p3d = p3d[:3] / p3d[3]+ return p3d+++def get_error_dict(errors_full, min_points=10):+ n_cams = errors_full.shape[0]+ errors_norm = np.linalg.norm(errors_full, axis=2)++ good = ~np.isnan(errors_full[:, :, 0])++ error_dict = dict()++ for i in range(n_cams):+ for j in range(i+1, n_cams):+ subset = good[i] & good[j]+ err_subset = errors_norm[:, subset][[i, j]]+ err_subset_mean = np.mean(err_subset, axis=0)+ if np.sum(subset) > min_points:+ percents = np.percentile(err_subset_mean, [15, 75])+ # percents = np.percentile(err_subset, [25, 75])+ error_dict[(i, j)] = (err_subset.shape[1], percents)+ return error_dict++def check_errors(cgroup, imgp):+ p3ds = cgroup.triangulate(imgp)+ errors_full = cgroup.reprojection_error(p3ds, imgp, mean=False)+ return get_error_dict(errors_full)++def subset_extra(extra, ixs):+ if extra is None:+ return None++ new_extra = {+ 'objp': extra['objp'][ixs],+ 'ids': extra['ids'][ixs],+ 'rvecs': extra['rvecs'][:, ixs],+ 'tvecs': extra['tvecs'][:, ixs]+ }+ return new_extra++def resample_points_extra(imgp, extra, n_samp=25):+ n_cams, n_points, _ = imgp.shape+ ids = remap_ids(extra['ids'])+ n_ids = np.max(ids)+1+ good = ~np.isnan(imgp[:, :, 0])+ ixs = np.arange(n_points)++ cam_counts = np.zeros((n_ids, n_cams), dtype='int64')+ for idnum in range(n_ids):+ cam_counts[idnum] = np.sum(good[:, ids == idnum], axis=1)+ cam_counts_random = cam_counts + np.random.random(size=cam_counts.shape)+ best_boards = np.argsort(-cam_counts_random, axis=0)++ cam_totals = np.zeros(n_cams, dtype='int64')++ include = set()+ for cam_num in range(n_cams):+ for board_id in best_boards[:, cam_num]:+ include.update(ixs[ids == board_id])+ cam_totals += cam_counts[board_id]+ if cam_totals[cam_num] >= n_samp or \+ cam_counts_random[board_id, cam_num] < 1:+ break++ final_ixs = sorted(include)+ newp = imgp[:, final_ixs]+ extra = subset_extra(extra, final_ixs)+ return newp, extra++def resample_points(imgp, extra=None, n_samp=25):+ # if extra is not None:+ # return resample_points_extra(imgp, extra, n_samp)++ n_cams = imgp.shape[0]+ good = ~np.isnan(imgp[:, :, 0])+ ixs = np.arange(imgp.shape[1])++ num_cams = np.sum(~np.isnan(imgp[:, :, 0]), axis=0)++ include = set()++ for i in range(n_cams):+ for j in range(i+1, n_cams):+ subset = good[i] & good[j]+ n_good = np.sum(subset)+ if n_good > 0:+ ## pick points, prioritizing points seen by more cameras+ arr = np.copy(num_cams[subset]).astype('float64')+ arr += np.random.random(size=arr.shape)+ picked_ix = np.argsort(-arr)[:n_samp]+ picked = ixs[subset][picked_ix]+ include.update(picked)++ final_ixs = sorted(include)+ newp = imgp[:, final_ixs]+ extra = subset_extra(extra, final_ixs)+ return newp, extra++def medfilt_data(values, size=15):+ padsize = size+5+ vpad = np.pad(values, (padsize, padsize), mode='reflect')+ vpadf = signal.medfilt(vpad, kernel_size=size)+ return vpadf[padsize:-padsize]++def nan_helper(y):+ return np.isnan(y), lambda z: z.nonzero()[0]++def interpolate_data(vals):+ nans, ix = nan_helper(vals)+ out = np.copy(vals)+ try:+ out[nans] = np.interp(ix(nans), ix(~nans), vals[~nans])+ except ValueError:+ out[:] = 0+ return out++def remap_ids(ids):+ unique_ids = np.unique(ids)+ ids_out = np.copy(ids)+ for i, num in enumerate(unique_ids):+ ids_out[ids == num] = i+ return ids_out++def transform_points(points, rvecs, tvecs):+ """Rotate points by given rotation vectors and translate.+ Rodrigues' rotation formula is used.+ """+ theta = np.linalg.norm(rvecs, axis=1)[:, np.newaxis]+ with np.errstate(invalid='ignore'):+ v = rvecs / theta+ v = np.nan_to_num(v)+ dot = np.sum(points * v, axis=1)[:, np.newaxis]+ cos_theta = np.cos(theta)+ sin_theta = np.sin(theta)++ rotated = cos_theta * points + \+ sin_theta * np.cross(v, points) + \+ dot * (1 - cos_theta) * v++ return rotated + tvecs+++class Camera:+ def __init__(self,+ matrix=np.eye(3),+ dist=np.zeros(5),+ size=None,+ rvec=np.zeros(3),+ tvec=np.zeros(3),+ name=None,+ extra_dist=False):++ self.set_camera_matrix(matrix)+ self.set_distortions(dist)+ self.set_size(size)+ self.set_rotation(rvec)+ self.set_translation(tvec)+ self.set_name(name)+ self.extra_dist = extra_dist++ def get_dict(self):+ return {+ 'name': self.get_name(),+ 'size': list(self.get_size()),+ 'matrix': self.get_camera_matrix().tolist(),+ 'distortions': self.get_distortions().tolist(),+ 'rotation': self.get_rotation().tolist(),+ 'translation': self.get_translation().tolist(),+ }++ def load_dict(self, d):+ self.set_camera_matrix(d['matrix'])+ self.set_rotation(d['rotation'])+ self.set_translation(d['translation'])+ self.set_distortions(d['distortions'])+ self.set_name(d['name'])+ self.set_size(d['size'])++ def from_dict(d):+ cam = Camera()+ cam.load_dict(d)+ return cam++ def get_camera_matrix(self):+ return self.matrix++ def get_distortions(self):+ return self.dist++ def set_camera_matrix(self, matrix):+ self.matrix = np.array(matrix, dtype='float64')++ def set_focal_length(self, fx, fy=None):+ if fy is None:+ fy = fx+ self.matrix[0, 0] = fx+ self.matrix[1, 1] = fy++ def get_focal_length(self, both=False):+ fx = self.matrix[0, 0]+ fy = self.matrix[1, 1]+ if both:+ return (fx, fy)+ else:+ return (fx + fy) / 2.0++ def set_distortions(self, dist):+ self.dist = np.array(dist, dtype='float64').ravel()++ def set_rotation(self, rvec):+ self.rvec = np.array(rvec, dtype='float64').ravel()++ def get_rotation(self):+ return self.rvec++ def set_translation(self, tvec):+ self.tvec = np.array(tvec, dtype='float64').ravel()++ def get_translation(self):+ return self.tvec++ def get_extrinsics_mat(self):+ return make_M(self.rvec, self.tvec)++ def get_name(self):+ return self.name++ def set_name(self, name):+ self.name = str(name)++ def set_size(self, size):+ """set size as (width, height)"""+ self.size = size++ def get_size(self):+ """get size as (width, height)"""+ return self.size++ def resize_camera(self, scale):+ """resize the camera by scale factor, updating intrinsics to match"""+ size = self.get_size()+ new_size = size[0] * scale, size[1] * scale+ matrix = self.get_camera_matrix()+ new_matrix = matrix * scale+ new_matrix[2, 2] = 1+ self.set_size(new_size)+ self.set_camera_matrix(new_matrix)++ def get_params(self):+ params = np.zeros(8 + self.extra_dist, dtype='float64')+ params[0:3] = self.get_rotation()+ params[3:6] = self.get_translation()+ params[6] = self.get_focal_length()+ dist = self.get_distortions()+ params[7] = dist[0]+ if self.extra_dist:+ params[8] = dist[1]+ return params++ def set_params(self, params):+ self.set_rotation(params[0:3])+ self.set_translation(params[3:6])+ self.set_focal_length(params[6])++ dist = np.zeros(5, dtype='float64')+ dist[0] = params[7]+ if self.extra_dist:+ dist[1] = params[8]+ self.set_distortions(dist)++ def distort_points(self, points):+ shape = points.shape+ points = points.reshape(-1, 1, 2)+ new_points = np.dstack([points, np.ones((points.shape[0], 1, 1))])+ out, _ = cv2.projectPoints(new_points, np.zeros(3), np.zeros(3),+ self.matrix.astype('float64'),+ self.dist.astype('float64'))+ return out.reshape(shape)++ def undistort_points(self, points):+ shape = points.shape+ points = points.reshape(-1, 1, 2)+ out = cv2.undistortPoints(points,+ self.matrix.astype('float64'),+ self.dist.astype('float64'))+ return out.reshape(shape)++ def project(self, points):+ points = points.reshape(-1, 1, 3)+ out, _ = cv2.projectPoints(points, self.rvec, self.tvec,+ self.matrix.astype('float64'),+ self.dist.astype('float64'))+ return out++ def reprojection_error(self, p3d, p2d):+ proj = self.project(p3d).reshape(p2d.shape)+ return p2d - proj++ def copy(self):+ return \+ Camera(matrix=self.get_camera_matrix().copy(),+ dist=self.get_distortions().copy(),+ size=self.get_size(),+ rvec=self.get_rotation().copy(),+ tvec=self.get_translation().copy(),+ name=self.get_name(),+ extra_dist=self.extra_dist)++class FisheyeCamera(Camera):+ def __init__(self,+ matrix=np.eye(3),+ dist=np.zeros(4),+ size=None,+ rvec=np.zeros(3),+ tvec=np.zeros(3),+ name=None,+ extra_dist=False):+ self.set_camera_matrix(matrix)+ self.set_distortions(dist)+ self.set_size(size)+ self.set_rotation(rvec)+ self.set_translation(tvec)+ self.set_name(name)+ self.extra_dist = extra_dist++ def from_dict(d):+ cam = FisheyeCamera()+ cam.load_dict(d)+ return cam++ def get_dict(self):+ d = super().get_dict()+ d['fisheye'] = True+ return d++ def distort_points(self, points):+ shape = points.shape+ points = points.reshape(-1, 1, 2)+ new_points = np.dstack([points, np.ones((points.shape[0], 1, 1))])+ out, _ = cv2.fisheye.projectPoints(new_points,+ np.zeros(3), np.zeros(3),+ self.matrix.astype('float64'),+ self.dist.astype('float64'))+ return out.reshape(shape)++ def undistort_points(self, points):+ shape = points.shape+ points = points.reshape(-1, 1, 2)+ out = cv2.fisheye.undistortPoints(points.astype('float64'),+ self.matrix.astype('float64'),+ self.dist.astype('float64'))+ return out.reshape(shape)++ def project(self, points):+ points = points.reshape(-1, 1, 3)+ out, _ = cv2.fisheye.projectPoints(points,+ self.rvec, self.tvec,+ self.matrix.astype('float64'),+ self.dist.astype('float64'))+ return out++ def set_params(self, params):+ self.set_rotation(params[0:3])+ self.set_translation(params[3:6])+ self.set_focal_length(params[6])++ dist = np.zeros(4, dtype='float64')+ dist[0] = params[7]+ if self.extra_dist:+ dist[1] = params[8]+ # dist[2] = params[9]+ # dist[3] = params[10]+ self.set_distortions(dist)++ def get_params(self):+ params = np.zeros(8+self.extra_dist, dtype='float64')+ params[0:3] = self.get_rotation()+ params[3:6] = self.get_translation()+ params[6] = self.get_focal_length()+ dist = self.get_distortions()+ params[7] = dist[0]+ if self.extra_dist:+ params[8] = dist[1]+ # params[9] = dist[2]+ # params[10] = dist[3]+ return params++ def copy(self):+ return FisheyeCamera(+ matrix=self.get_camera_matrix().copy(),+ dist=self.get_distortions().copy(),+ size=self.get_size(),+ rvec=self.get_rotation().copy(),+ tvec=self.get_translation().copy(),+ name=self.get_name(),+ extra_dist=self.extra_dist)++class CameraGroup:+ def __init__(self, cameras, metadata={}):+ self.cameras = cameras+ self.metadata = metadata++ def subset_cameras(self, indices):+ cams = [self.cameras[ix].copy() for ix in indices]+ return CameraGroup(cams, self.metadata)++ def subset_cameras_names(self, names):+ cur_names = self.get_names()+ cur_names_dict = dict(zip(cur_names, range(len(cur_names))))+ indices = []+ for name in names:+ if name not in cur_names_dict:+ raise IndexError(+ "name {} not part of camera names: {}".format(+ name, cur_names+ ))+ indices.append(cur_names_dict[name])+ return self.subset_cameras(indices)++ def project(self, points):+ """Given an Nx3 array of points, this returns an CxNx2 array of 2D points,+ where C is the number of cameras"""+ points = points.reshape(-1, 1, 3)+ n_points = points.shape[0]+ n_cams = len(self.cameras)++ out = np.empty((n_cams, n_points, 2), dtype='float64')+ for cnum, cam in enumerate(self.cameras):+ out[cnum] = cam.project(points).reshape(n_points, 2)++ return out++ def triangulate(self, points, undistort=True, progress=False):+ """Given an CxNx2 array, this returns an Nx3 array of points,+ where N is the number of points and C is the number of cameras"""++ assert points.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), points.shape+ )++ one_point = False+ if len(points.shape) == 2:+ points = points.reshape(-1, 1, 2)+ one_point = True++ if undistort:+ new_points = np.empty(points.shape)+ for cnum, cam in enumerate(self.cameras):+ # must copy in order to satisfy opencv underneath+ sub = np.copy(points[cnum])+ new_points[cnum] = cam.undistort_points(sub)+ points = new_points++ n_cams, n_points, _ = points.shape++ out = np.empty((n_points, 3))+ out[:] = np.nan++ cam_mats = np.array([cam.get_extrinsics_mat() for cam in self.cameras])++ if progress:+ iterator = trange(n_points, ncols=70)+ else:+ iterator = range(n_points)++ for ip in iterator:+ subp = points[:, ip, :]+ good = ~np.isnan(subp[:, 0])+ if np.sum(good) >= 2:+ out[ip] = triangulate_simple(subp[good], cam_mats[good])++ if one_point:+ out = out[0]++ return out++ def triangulate_possible(self, points, undistort=True,+ min_cams=2, progress=False, threshold=0.5):+ """Given an CxNxPx2 array, this returns an Nx3 array of points+ by triangulating all possible points and picking the ones with+ best reprojection error+ where:+ C: number of cameras+ N: number of points+ P: number of possible options per point+ """++ assert points.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), points.shape+ )++ n_cams, n_points, n_possible, _ = points.shape++ cam_nums, point_nums, possible_nums = np.where(+ ~np.isnan(points[:, :, :, 0]))++ all_iters = defaultdict(dict)++ for cam_num, point_num, possible_num in zip(cam_nums, point_nums,+ possible_nums):+ if cam_num not in all_iters[point_num]:+ all_iters[point_num][cam_num] = []+ all_iters[point_num][cam_num].append((cam_num, possible_num))++ for point_num in all_iters.keys():+ for cam_num in all_iters[point_num].keys():+ all_iters[point_num][cam_num].append(None)++ out = np.full((n_points, 3), np.nan, dtype='float64')+ picked_vals = np.zeros((n_cams, n_points, n_possible), dtype='bool')+ errors = np.zeros(n_points, dtype='float64')+ points_2d = np.full((n_cams, n_points, 2), np.nan, dtype='float64')++ if progress:+ iterator = trange(n_points, ncols=70)+ else:+ iterator = range(n_points)++ for point_ix in iterator:+ best_point = None+ best_error = 200++ n_cams_max = len(all_iters[point_ix])++ for picked in itertools.product(*all_iters[point_ix].values()):+ picked = [p for p in picked if p is not None]+ if len(picked) < min_cams and len(picked) != n_cams_max:+ continue++ cnums = [p[0] for p in picked]+ xnums = [p[1] for p in picked]++ pts = points[cnums, point_ix, xnums]+ cc = self.subset_cameras(cnums)++ p3d = cc.triangulate(pts, undistort=undistort)+ err = cc.reprojection_error(p3d, pts, mean=True)++ if err < best_error:+ best_point = {+ 'error': err,+ 'point': p3d[:3],+ 'points': pts,+ 'picked': picked,+ 'joint_ix': point_ix+ }+ best_error = err+ if best_error < threshold:+ break++ if best_point is not None:+ out[point_ix] = best_point['point']+ picked = best_point['picked']+ cnums = [p[0] for p in picked]+ xnums = [p[1] for p in picked]+ picked_vals[cnums, point_ix, xnums] = True+ errors[point_ix] = best_point['error']+ points_2d[cnums, point_ix] = best_point['points']++ return out, picked_vals, points_2d, errors++ def triangulate_ransac(self, points, undistort=True, min_cams=2, progress=False):+ """Given an CxNx2 array, this returns an Nx3 array of points,+ where N is the number of points and C is the number of cameras"""++ assert points.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), points.shape+ )++ n_cams, n_points, _ = points.shape++ points_ransac = points.reshape(n_cams, n_points, 1, 2)++ return self.triangulate_possible(points_ransac,+ undistort=undistort,+ min_cams=min_cams,+ progress=progress)+++ @jit(nopython=True, parallel=True, forceobj=True)+ def reprojection_error(self, p3ds, p2ds, mean=False):+ """Given an Nx3 array of 3D points and an CxNx2 array of 2D points,+ where N is the number of points and C is the number of cameras,+ this returns an CxNx2 array of errors.+ Optionally mean=True, this averages the errors and returns array of length N of errors"""++ one_point = False+ if len(p3ds.shape) == 1 and len(p2ds.shape) == 2:+ p3ds = p3ds.reshape(1, 3)+ p2ds = p2ds.reshape(-1, 1, 2)+ one_point = True++ n_cams, n_points, _ = p2ds.shape+ assert p3ds.shape == (n_points, 3), \+ "shapes of 2D and 3D points are not consistent: " \+ "2D={}, 3D={}".format(p2ds.shape, p3ds.shape)++ errors = np.empty((n_cams, n_points, 2))++ for cnum, cam in enumerate(self.cameras):+ errors[cnum] = cam.reprojection_error(p3ds, p2ds[cnum])++ if mean:+ errors_norm = np.linalg.norm(errors, axis=2)+ good = ~np.isnan(errors_norm)+ errors_norm[~good] = 0+ denom = np.sum(good, axis=0).astype('float64')+ denom[denom < 1.5] = np.nan+ errors = np.sum(errors_norm, axis=0) / denom++ if one_point:+ if mean:+ errors = float(errors[0])+ else:+ errors = errors.reshape(-1, 2)++ return errors+++ def bundle_adjust_iter(self, p2ds, extra=None,+ n_iters=10, start_mu=15, end_mu=1,+ max_nfev=200, ftol=1e-4,+ n_samp_iter=100, n_samp_full=1000,+ error_threshold=0.3,+ verbose=False):+ """Given an CxNx2 array of 2D points,+ where N is the number of points and C is the number of cameras,+ this performs iterative bundle adjustsment to fine-tune the parameters of the cameras.+ That is, it performs bundle adjustment multiple times, adjusting the weights given to points+ to reduce the influence of outliers.+ This is inspired by the algorithm for Fast Global Registration by Zhou, Park, and Koltun+ """++ assert p2ds.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), p2ds.shape+ )++ p2ds_full = p2ds+ extra_full = extra++ p2ds, extra = resample_points(p2ds_full, extra_full,+ n_samp=n_samp_full)+ error = self.average_error(p2ds, median=True)++ if verbose:+ print('error: ', error)++ mus = np.exp(np.linspace(np.log(start_mu), np.log(end_mu), num=n_iters))++ if verbose:+ print('n_samples: {}'.format(n_samp_iter))++ for i in range(n_iters):+ p2ds, extra = resample_points(p2ds_full, extra_full,+ n_samp=n_samp_full)+ p3ds = self.triangulate(p2ds)+ errors_full = self.reprojection_error(p3ds, p2ds, mean=False)+ errors_norm = self.reprojection_error(p3ds, p2ds, mean=True)++ error_dict = get_error_dict(errors_full)+ max_error = 0+ min_error = 0+ for k, v in error_dict.items():+ num, percents = v+ max_error = max(percents[-1], max_error)+ min_error = max(percents[0], min_error)+ mu = max(min(max_error, mus[i]), min_error)++ good = errors_norm < mu+ extra_good = subset_extra(extra, good)+ p2ds_samp, extra_samp = resample_points(+ p2ds[:, good], extra_good, n_samp=n_samp_iter)++ error = np.median(errors_norm)++ if error < error_threshold:+ break++ if verbose:+ pprint(error_dict)+ print('error: {:.2f}, mu: {:.1f}, ratio: {:.3f}'.format(error, mu, np.mean(good)))++ self.bundle_adjust(p2ds_samp, extra_samp,+ loss='linear', ftol=ftol,+ max_nfev=max_nfev,+ verbose=verbose)+++ p2ds, extra = resample_points(p2ds_full, extra_full,+ n_samp=n_samp_full)+ p3ds = self.triangulate(p2ds)+ errors_full = self.reprojection_error(p3ds, p2ds, mean=False)+ errors_norm = self.reprojection_error(p3ds, p2ds, mean=True)+ error_dict = get_error_dict(errors_full)+ if verbose:+ pprint(error_dict)++ max_error = 0+ min_error = 0+ for k, v in error_dict.items():+ num, percents = v+ max_error = max(percents[-1], max_error)+ min_error = max(percents[0], min_error)+ mu = max(max(max_error, end_mu), min_error)++ good = errors_norm < mu+ extra_good = subset_extra(extra, good)+ self.bundle_adjust(p2ds[:, good], extra_good,+ loss='linear',+ ftol=ftol, max_nfev=max(200, max_nfev),+ verbose=verbose)++ error = self.average_error(p2ds, median=True)++ p3ds = self.triangulate(p2ds)+ errors_full = self.reprojection_error(p3ds, p2ds, mean=False)+ error_dict = get_error_dict(errors_full)+ if verbose:+ pprint(error_dict)++ if verbose:+ print('error: ', error)++ return error++ def bundle_adjust(self, p2ds, extra=None,+ loss='linear',+ threshold=50,+ ftol=1e-4,+ max_nfev=1000,+ weights=None,+ start_params=None,+ verbose=True):+ """Given an CxNx2 array of 2D points,+ where N is the number of points and C is the number of cameras,+ this performs bundle adjustsment to fine-tune the parameters of the cameras"""++ assert p2ds.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), p2ds.shape+ )++ if extra is not None:+ extra['ids_map'] = remap_ids(extra['ids'])++ x0, n_cam_params = self._initialize_params_bundle(p2ds, extra)++ if start_params is not None:+ x0 = start_params+ n_cam_params = len(self.cameras[0].get_params())++ error_fun = self._error_fun_bundle++ jac_sparse = self._jac_sparsity_bundle(p2ds, n_cam_params, extra)++ f_scale = threshold+ opt = optimize.least_squares(error_fun,+ x0,+ jac_sparsity=jac_sparse,+ f_scale=f_scale,+ x_scale='jac',+ loss=loss,+ ftol=ftol,+ method='trf',+ tr_solver='lsmr',+ verbose=2 * verbose,+ max_nfev=max_nfev,+ args=(p2ds, n_cam_params, extra))+ best_params = opt.x++ for i, cam in enumerate(self.cameras):+ a = i * n_cam_params+ b = (i + 1) * n_cam_params+ cam.set_params(best_params[a:b])++ error = self.average_error(p2ds)+ return error++ @jit(nopython=True, parallel=True, forceobj=True)+ def _error_fun_bundle(self, params, p2ds, n_cam_params, extra):+ """Error function for bundle adjustment"""+ good = ~np.isnan(p2ds)+ n_cams = len(self.cameras)++ for i in range(n_cams):+ cam = self.cameras[i]+ a = i * n_cam_params+ b = (i + 1) * n_cam_params+ cam.set_params(params[a:b])++ n_cams = len(self.cameras)+ sub = n_cam_params * n_cams+ n3d = p2ds.shape[1] * 3+ p3ds_test = params[sub:sub+n3d].reshape(-1, 3)+ errors = self.reprojection_error(p3ds_test, p2ds)+ errors_reproj = errors[good]++ if extra is not None:+ ids = extra['ids_map']+ objp = extra['objp']+ min_scale = np.min(objp[objp > 0])+ n_boards = int(np.max(ids)) + 1+ a = sub+n3d+ rvecs = params[a:a+n_boards*3].reshape(-1, 3)+ tvecs = params[a+n_boards*3:a+n_boards*6].reshape(-1, 3)+ expected = transform_points(objp, rvecs[ids], tvecs[ids])+ errors_obj = 2 * (p3ds_test - expected).ravel() / min_scale+ else:+ errors_obj = np.array([])++ return np.hstack([errors_reproj, errors_obj])+++ def _jac_sparsity_bundle(self, p2ds, n_cam_params, extra):+ """Given an CxNx2 array of 2D points,+ where N is the number of points and C is the number of cameras,+ compute the sparsity structure of the jacobian for bundle adjustment"""++ point_indices = np.zeros(p2ds.shape, dtype='int32')+ cam_indices = np.zeros(p2ds.shape, dtype='int32')++ for i in range(p2ds.shape[1]):+ point_indices[:, i] = i++ for j in range(p2ds.shape[0]):+ cam_indices[j] = j++ good = ~np.isnan(p2ds)++ if extra is not None:+ ids = extra['ids_map']+ n_boards = int(np.max(ids)) + 1+ total_board_params = n_boards * (3 + 3) # rvecs + tvecs+ else:+ n_boards = 0+ total_board_params = 0++ n_cams = p2ds.shape[0]+ n_points = p2ds.shape[1]+ total_params_reproj = n_cams * n_cam_params + n_points * 3+ n_params = total_params_reproj + total_board_params++ n_good_values = np.sum(good)+ if extra is not None:+ n_errors = n_good_values + n_points * 3+ else:+ n_errors = n_good_values++ A_sparse = dok_matrix((n_errors, n_params), dtype='int16')++ cam_indices_good = cam_indices[good]+ point_indices_good = point_indices[good]++ # -- reprojection error --+ ix = np.arange(n_good_values)++ ## update camera params based on point error+ for i in range(n_cam_params):+ A_sparse[ix, cam_indices_good * n_cam_params + i] = 1++ ## update point position based on point error+ for i in range(3):+ A_sparse[ix, n_cams * n_cam_params + point_indices_good * 3 + i] = 1++ # -- match for the object points--+ if extra is not None:+ point_ix = np.arange(n_points)++ ## update all the camera parameters+ # A_sparse[n_good_values:n_good_values+n_points*3,+ # 0:n_cams*n_cam_params] = 1++ ## update board rotation and translation based on error from expected+ for i in range(3):+ for j in range(3):+ A_sparse[n_good_values + point_ix*3 + i,+ total_params_reproj + ids*3 + j] = 1+ A_sparse[n_good_values + point_ix*3 + i,+ total_params_reproj + n_boards*3 + ids*3 + j] = 1+++ ## update point position based on error from expected+ for i in range(3):+ A_sparse[n_good_values + point_ix*3 + i,+ n_cams*n_cam_params + point_ix*3 + i] = 1+++ return A_sparse++ def _initialize_params_bundle(self, p2ds, extra):+ """Given an CxNx2 array of 2D points,+ where N is the number of points and C is the number of cameras,+ initializes the parameters for bundle adjustment"""++ cam_params = np.hstack([cam.get_params() for cam in self.cameras])+ n_cam_params = len(cam_params) // len(self.cameras)++ total_cam_params = len(cam_params)++ n_cams, n_points, _ = p2ds.shape+ assert n_cams == len(self.cameras), \+ "number of cameras in CameraGroup does not " \+ "match number of cameras in 2D points given"++ p3ds = self.triangulate(p2ds)++ if extra is not None:+ ids = extra['ids_map']+ n_boards = int(np.max(ids[~np.isnan(ids)])) + 1+ total_board_params = n_boards * (3 + 3) # rvecs + tvecs++ # initialize to 0+ rvecs = np.zeros((n_boards, 3), dtype='float64')+ tvecs = np.zeros((n_boards, 3), dtype='float64')++ if 'rvecs' in extra and 'tvecs' in extra:+ rvecs_all = extra['rvecs']+ tvecs_all = extra['tvecs']+ for board_num in range(n_boards):+ point_id = np.where(ids == board_num)[0][0]+ cam_ids_possible = np.where(~np.isnan(p2ds[:, point_id, 0]))[0]+ cam_id = np.random.choice(cam_ids_possible)+ M_cam = self.cameras[cam_id].get_extrinsics_mat()+ M_board_cam = make_M(rvecs_all[cam_id, point_id],+ tvecs_all[cam_id, point_id])+ M_board = np.matmul(np.linalg.inv(M_cam), M_board_cam)+ rvec, tvec = get_rtvec(M_board)+ rvecs[board_num] = rvec+ tvecs[board_num] = tvec+++ else:+ total_board_params = 0++ x0 = np.zeros(total_cam_params + p3ds.size + total_board_params)+ x0[:total_cam_params] = cam_params+ x0[total_cam_params:total_cam_params+p3ds.size] = p3ds.ravel()++ if extra is not None:+ start_board = total_cam_params+p3ds.size+ x0[start_board:start_board + n_boards*3] = rvecs.ravel()+ x0[start_board + n_boards*3:start_board + n_boards*6] = \+ tvecs.ravel()++ return x0, n_cam_params++ def optim_points(self, points, p3ds,+ constraints=[],+ constraints_weak=[],+ scale_smooth=4,+ scale_length=2, scale_length_weak=0.5,+ reproj_error_threshold=15, reproj_loss='soft_l1',+ n_deriv_smooth=1, scores=None, verbose=False):+ """+ Take in an array of 2D points of shape CxNxJx2,+ an array of 3D points of shape NxJx3,+ and an array of constraints of shape Kx2, where+ C: number of camera+ N: number of frames+ J: number of joints+ K: number of constraints++ This function creates an optimized array of 3D points of shape NxJx3.++ Example constraints:+ constraints = [[0, 1], [1, 2], [2, 3]]+ (meaning that lengths of segments 0->1, 1->2, 2->3 are all constant)++ """+ assert points.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), points.shape+ )++ n_cams, n_frames, n_joints, _ = points.shape+ constraints = np.array(constraints)+ constraints_weak = np.array(constraints_weak)++ p3ds_intp = np.apply_along_axis(interpolate_data, 0, p3ds)++ p3ds_med = np.apply_along_axis(medfilt_data, 0, p3ds_intp, size=7)++ default_smooth = 1.0/np.mean(np.abs(np.diff(p3ds_med, axis=0)))+ scale_smooth_full = scale_smooth * default_smooth++ t1 = time.time()++ x0 = self._initialize_params_triangulation(+ p3ds_intp, constraints, constraints_weak)++ x0[~np.isfinite(x0)] = 0++ jac = self._jac_sparsity_triangulation(+ points, constraints, constraints_weak, n_deriv_smooth)++ opt2 = optimize.least_squares(self._error_fun_triangulation,+ x0=x0, jac_sparsity=jac,+ loss='linear',+ ftol=1e-3,+ verbose=2*verbose,+ args=(points,+ constraints,+ constraints_weak,+ scores,+ scale_smooth_full,+ scale_length,+ scale_length_weak,+ reproj_error_threshold,+ reproj_loss,+ n_deriv_smooth))++ p3ds_new2 = opt2.x[:p3ds.size].reshape(p3ds.shape)++ t2 = time.time()++ if verbose:+ print('optimization took {:.2f} seconds'.format(t2 - t1))++ return p3ds_new2+++ def optim_points_possible(self, points, p3ds,+ constraints=[],+ constraints_weak=[],+ scale_smooth=4,+ scale_length=2, scale_length_weak=0.5,+ reproj_error_threshold=15, reproj_loss='soft_l1',+ n_deriv_smooth=1, scores=None, verbose=False):+ """+ Take in an array of 2D points of shape CxNxJxPx2,+ an array of 3D points of shape NxJx3,+ and an array of constraints of shape Kx2, where+ C: number of camera+ N: number of frames+ J: number of joints+ P: number of possible options per point+ K: number of constraints++ This function creates an optimized array of 3D points of shape NxJx3.++ Example constraints:+ constraints = [[0, 1], [1, 2], [2, 3]]+ (meaning that lengths of segments 0->1, 1->2, 2->3 are all constant)++ """+ assert points.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), points.shape+ )++ n_cams, n_frames, n_joints, n_possible, _ = points.shape+ constraints = np.array(constraints)+ constraints_weak = np.array(constraints_weak)++ p3ds_intp = np.apply_along_axis(interpolate_data, 0, p3ds)++ p3ds_med = np.apply_along_axis(medfilt_data, 0, p3ds_intp, size=7)++ default_smooth = 1.0/np.mean(np.abs(np.diff(p3ds_med, axis=0)))+ scale_smooth_full = scale_smooth * default_smooth++ t1 = time.time()++ x0 = self._initialize_params_triangulation_possible(+ p3ds_intp, points, constraints=constraints, constraints_weak=constraints_weak)++ print('getting jacobian...')+ jac = self._jac_sparsity_triangulation_possible(+ points,+ constraints=constraints,+ constraints_weak=constraints_weak,+ n_deriv_smooth=n_deriv_smooth)++ beta = 5++ print('starting optimization...')+ opt2 = optimize.least_squares(self._error_fun_triangulation_possible,+ x0=x0, jac_sparsity=jac,+ loss='linear',+ ftol=1e-3,+ verbose=2*verbose,+ args=(points,+ beta,+ constraints,+ constraints_weak,+ scores,+ scale_smooth_full,+ scale_length,+ scale_length_weak,+ reproj_error_threshold,+ reproj_loss,+ n_deriv_smooth))+ params = opt2.x++ p3ds_new2 = params[:p3ds.size].reshape(p3ds.shape)++ bad = np.isnan(points[:, :, :, :, 0])+ all_bad = np.all(bad, axis=3)++ n_params_norm = p3ds.size + len(constraints) + len(constraints_weak)++ alphas = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='float64')+ alphas[~bad] = params[n_params_norm:]++ alphas_exp = np.exp(beta * alphas)+ alphas_exp[bad] = 0+ alphas_sum = np.sum(alphas_exp, axis=3)+ alphas_sum[all_bad] = 1+ alphas_norm = alphas_exp / alphas_sum[:, :, :, None]+ alphas_norm[bad] = np.nan++ t2 = time.time()++ if verbose:+ print('optimization took {:.2f} seconds'.format(t2 - t1))++ return p3ds_new2, alphas_norm+++ def triangulate_optim(self, points, init_ransac=False, init_progress=False,+ **kwargs):+ """+ Take in an array of 2D points of shape CxNxJx2, and an array of constraints of shape Kx2, where+ C: number of camera+ N: number of frames+ J: number of joints+ K: number of constraints++ This function creates an optimized array of 3D points of shape NxJx3.++ Example constraints:+ constraints = [[0, 1], [1, 2], [2, 3]]+ (meaning that lengths of segments 0->1, 1->2, 2->3 are all constant)++ """++ assert points.shape[0] == len(self.cameras), \+ "Invalid points shape, first dim should be equal to" \+ " number of cameras ({}), but shape is {}".format(+ len(self.cameras), points.shape+ )++ n_cams, n_frames, n_joints, _ = points.shape+ # constraints = np.array(constraints)+ # constraints_weak = np.array(constraints_weak)++ points_shaped = points.reshape(n_cams, n_frames*n_joints, 2)+ if init_ransac:+ p3ds, picked, p2ds, errors = self.triangulate_ransac(points_shaped, progress=init_progress)+ points = p2ds.reshape(points.shape)+ else:+ p3ds = self.triangulate(points_shaped, progress=init_progress)+ p3ds = p3ds.reshape((n_frames, n_joints, 3))++ c = np.isfinite(p3ds[:, :, 0])+ if np.sum(c) < 20:+ print("warning: not enough 3D points to run optimization")+ return p3ds++ return self.optim_points(points, p3ds, **kwargs)++++ @jit(nopython=True, forceobj=True, parallel=True)+ def _error_fun_triangulation(self, params, p2ds,+ constraints=[],+ constraints_weak=[],+ scores=None,+ scale_smooth=10000,+ scale_length=1,+ scale_length_weak=0.2,+ reproj_error_threshold=100,+ reproj_loss='soft_l1',+ n_deriv_smooth=1):+ n_cams, n_frames, n_joints, _ = p2ds.shape++ n_3d = n_frames*n_joints*3+ n_constraints = len(constraints)+ n_constraints_weak = len(constraints_weak)++ # load params+ p3ds = params[:n_3d].reshape((n_frames, n_joints, 3))+ joint_lengths = np.array(params[n_3d:n_3d+n_constraints])+ joint_lengths_weak = np.array(params[n_3d+n_constraints:])++ # reprojection errors+ p3ds_flat = p3ds.reshape(-1, 3)+ p2ds_flat = p2ds.reshape((n_cams, -1, 2))+ errors = self.reprojection_error(p3ds_flat, p2ds_flat)+ if scores is not None:+ scores_flat = scores.reshape((n_cams, -1))+ errors = errors * scores_flat[:, :, None]+ errors_reproj = errors[~np.isnan(p2ds_flat)]++ rp = reproj_error_threshold+ errors_reproj = np.abs(errors_reproj)+ if reproj_loss == 'huber':+ bad = errors_reproj > rp+ errors_reproj[bad] = rp*(2*np.sqrt(errors_reproj[bad]/rp) - 1)+ elif reproj_loss == 'linear':+ pass+ elif reproj_loss == 'soft_l1':+ errors_reproj = rp*2*(np.sqrt(1+errors_reproj/rp)-1)++ # temporal constraint+ errors_smooth = np.diff(p3ds, n=n_deriv_smooth, axis=0).ravel() * scale_smooth++ # joint length constraint+ errors_lengths = np.empty((n_constraints, n_frames), dtype='float64')+ for cix, (a, b) in enumerate(constraints):+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)+ expected = joint_lengths[cix]+ errors_lengths[cix] = 100*(lengths - expected)/expected+ errors_lengths = errors_lengths.ravel() * scale_length++ errors_lengths_weak = np.empty((n_constraints_weak, n_frames), dtype='float64')+ for cix, (a, b) in enumerate(constraints_weak):+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)+ expected = joint_lengths_weak[cix]+ errors_lengths_weak[cix] = 100*(lengths - expected)/expected+ errors_lengths_weak = errors_lengths_weak.ravel() * scale_length_weak++ return np.hstack([errors_reproj, errors_smooth,+ errors_lengths, errors_lengths_weak])++ def _error_fun_triangulation_possible(self, params, p2ds,+ beta=2,+ constraints=[],+ constraints_weak=[],+ *args):+ # extract alphas from end of params+ # soft argmax for picking the appropriate points from p2ds+ # pass the points to error_fun_triangulate_possible for residuals+ # add errors to keep the alphas in check+ # return all the errors++ n_cams, n_frames, n_joints, n_possible, _ = p2ds.shape++ n_3d = n_frames*n_joints*3+ n_constraints = len(constraints)+ n_constraints_weak = len(constraints_weak)+ n_params_norm = n_3d+n_constraints+n_constraints_weak++ # load params+ bad = np.isnan(p2ds[:, :, :, :, 0])+ all_bad = np.all(bad, axis=3)++ alphas = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='float64')+ alphas[~bad] = params[n_params_norm:]+ params_rest = np.array(params[:n_params_norm])++ # get normalized alphas+ alphas_exp = np.exp(beta * alphas)+ alphas_exp[bad] = 0+ alphas_sum = np.sum(alphas_exp, axis=3)+ alphas_sum[all_bad] = 1+ alphas_norm = alphas_exp / alphas_sum[:, :, :, None]++ # extract the 2D points using soft argmax+ p2ds_test = np.copy(p2ds)+ p2ds_test[bad] = 0+ p2ds_adj = np.sum(alphas_norm[:, :, :, :, None] * p2ds_test, axis=3)+ p2ds_adj[all_bad] = np.nan++ errors = self._error_fun_triangulation(params_rest, p2ds_adj,+ constraints, constraints_weak, *args)++ alphas_test = alphas_norm[~all_bad]+ errors_alphas = (1 - np.std(alphas_test, axis=1)) * 10++ return np.hstack([errors, errors_alphas])+++ def _initialize_params_triangulation(self, p3ds,+ constraints=[],+ constraints_weak=[]):+ joint_lengths = np.empty(len(constraints), dtype='float64')+ joint_lengths_weak = np.empty(len(constraints_weak), dtype='float64')++ for cix, (a, b) in enumerate(constraints):+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)+ joint_lengths[cix] = np.median(lengths)+++ for cix, (a, b) in enumerate(constraints_weak):+ lengths = np.linalg.norm(p3ds[:, a] - p3ds[:, b], axis=1)+ joint_lengths_weak[cix] = np.median(lengths)++ all_lengths = np.hstack([joint_lengths, joint_lengths_weak])+ med = np.median(all_lengths)+ if med == 0:+ med = 1e-3++ mad = np.median(np.abs(all_lengths - med))++ joint_lengths[joint_lengths == 0] = med+ joint_lengths_weak[joint_lengths_weak == 0] = med+ joint_lengths[joint_lengths > med+mad*5] = med+ joint_lengths_weak[joint_lengths_weak > med+mad*5] = med++ return np.hstack([p3ds.ravel(), joint_lengths, joint_lengths_weak])++ def _initialize_params_triangulation_possible(self, p3ds, p2ds, **kwargs):+ # initialize params using above function+ # initialize alphas to 1 for first one and 0 for other possible++ n_cams, n_frames, n_joints, n_possible, _ = p2ds.shape+ good = ~np.isnan(p2ds[:, :, :, :, 0])++ alphas = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='float64')+ alphas[:, :, :, 0] = 0++ params = self._initialize_params_triangulation(p3ds, **kwargs)+ params_full = np.hstack([params, alphas[good]])++ return params_full++ def _jac_sparsity_triangulation(self, p2ds,+ constraints=[],+ constraints_weak=[],+ n_deriv_smooth=1):+ n_cams, n_frames, n_joints, _ = p2ds.shape+ n_constraints = len(constraints)+ n_constraints_weak = len(constraints_weak)++ p2ds_flat = p2ds.reshape((n_cams, -1, 2))++ point_indices = np.zeros(p2ds_flat.shape, dtype='int32')+ for i in range(p2ds_flat.shape[1]):+ point_indices[:, i] = i++ point_indices_3d = np.arange(n_frames*n_joints)\+ .reshape((n_frames, n_joints))++ good = ~np.isnan(p2ds_flat)+ n_errors_reproj = np.sum(good)+ n_errors_smooth = (n_frames-n_deriv_smooth) * n_joints * 3+ n_errors_lengths = n_constraints * n_frames+ n_errors_lengths_weak = n_constraints_weak * n_frames++ n_errors = n_errors_reproj + n_errors_smooth + \+ n_errors_lengths + n_errors_lengths_weak++ n_3d = n_frames*n_joints*3+ n_params = n_3d + n_constraints + n_constraints_weak++ point_indices_good = point_indices[good]++ A_sparse = dok_matrix((n_errors, n_params), dtype='int16')++ # constraints for reprojection errors+ ix_reproj = np.arange(n_errors_reproj)+ for k in range(3):+ A_sparse[ix_reproj, point_indices_good * 3 + k] = 1++ # sparse constraints for smoothness in time+ frames = np.arange(n_frames-n_deriv_smooth)+ for j in range(n_joints):+ for n in range(n_deriv_smooth+1):+ pa = point_indices_3d[frames, j]+ pb = point_indices_3d[frames+n, j]+ for k in range(3):+ A_sparse[n_errors_reproj + pa*3 + k, pb*3 + k] = 1++ ## -- strong constraints --+ # joint lengths should change with joint lengths errors+ start = n_errors_reproj + n_errors_smooth+ frames = np.arange(n_frames)+ for cix, (a, b) in enumerate(constraints):+ A_sparse[start + cix*n_frames + frames, n_3d+cix] = 1++ # points should change accordingly to match joint lengths too+ frames = np.arange(n_frames)+ for cix, (a, b) in enumerate(constraints):+ pa = point_indices_3d[frames, a]+ pb = point_indices_3d[frames, b]+ for k in range(3):+ A_sparse[start + cix*n_frames + frames, pa*3 + k] = 1+ A_sparse[start + cix*n_frames + frames, pb*3 + k] = 1++ ## -- weak constraints --+ # joint lengths should change with joint lengths errors+ start = n_errors_reproj + n_errors_smooth + n_errors_lengths+ frames = np.arange(n_frames)+ for cix, (a, b) in enumerate(constraints_weak):+ A_sparse[start + cix*n_frames + frames, n_3d + n_constraints + cix] = 1++ # points should change accordingly to match joint lengths too+ frames = np.arange(n_frames)+ for cix, (a, b) in enumerate(constraints_weak):+ pa = point_indices_3d[frames, a]+ pb = point_indices_3d[frames, b]+ for k in range(3):+ A_sparse[start + cix*n_frames + frames, pa*3 + k] = 1+ A_sparse[start + cix*n_frames + frames, pb*3 + k] = 1++ return A_sparse++ def _jac_sparsity_triangulation_possible(self, p2ds_full, **kwargs):+ # initialize sparse jacobian using above function+ # extend to include alphas from parameters+ ## TODO: this initialization is really slow for some reason++ n_cams, n_frames, n_joints, n_possible, _ = p2ds_full.shape+ good_full = ~np.isnan(p2ds_full[:, :, :, :, 0])+ any_good = np.any(good_full, axis=3)++ n_alphas = np.sum(good_full)+ n_errors_alphas = np.sum(any_good)++ p2ds = p2ds_full[:, :, :, 0]+ A_sparse = self._jac_sparsity_triangulation(p2ds, **kwargs)++ n_errors, n_params = A_sparse.shape++ B_sparse = dok_matrix((n_errors + n_errors_alphas, n_params + n_alphas), dtype='int16')+ for r, c in zip(*A_sparse.nonzero()):+ B_sparse[r, c] = A_sparse[r, c]++ point_indices_2d = np.arange(n_cams*n_frames*n_joints)\+ .reshape(n_cams, n_frames, n_joints)+ point_indices_2d_rep = np.repeat(point_indices_2d[:, :, :, None], 2, axis=3)+ point_indices_2d_good = point_indices_2d_rep[~np.isnan(p2ds)]+ point_indices_good = point_indices_2d[any_good]++ alpha_indices = np.zeros((n_cams, n_frames, n_joints, n_possible), dtype='int64')+ for pnum in range(n_possible):+ alpha_indices[:, :, :, pnum] = point_indices_2d++ alpha_indices_good = alpha_indices[good_full]++ # alphas should change according to the reprojection error for each corresponding point+ point_indices_2d_good_find = defaultdict(list)+ for ix, p in enumerate(point_indices_2d_good):+ point_indices_2d_good_find[p].append(ix)++ for ix, alpha_index in enumerate(alpha_indices_good):+ B_sparse[point_indices_2d_good_find[alpha_index],+ n_params + ix] = 1++ # alphas should change according to the alpha errors+ point_indices_good_find = dict()+ for ix, p in enumerate(point_indices_good):+ point_indices_good_find[p] = ix++ for ix, alpha_index in enumerate(alpha_indices_good):+ if alpha_index in point_indices_good_find:+ err_ix = n_errors + point_indices_good_find[alpha_index]+ B_sparse[err_ix, n_params + ix] = 1++ return B_sparse++ def copy(self):+ cameras = [cam.copy() for cam in self.cameras]+ metadata = copy(self.metadata)+ return CameraGroup(cameras, metadata)++ def set_rotations(self, rvecs):+ for cam, rvec in zip(self.cameras, rvecs):+ cam.set_rotation(rvec)++ def set_translations(self, tvecs):+ for cam, tvec in zip(self.cameras, tvecs):+ cam.set_translation(tvec)++ def get_rotations(self):+ rvecs = []+ for cam in self.cameras:+ rvec = cam.get_rotation()+ rvecs.append(rvec)+ return np.array(rvecs)++ def get_translations(self):+ tvecs = []+ for cam in self.cameras:+ tvec = cam.get_translation()+ tvecs.append(tvec)+ return np.array(tvecs)++ def get_names(self):+ return [cam.get_name() for cam in self.cameras]++ def set_names(self, names):+ for cam, name in zip(self.cameras, names):+ cam.set_name(name)++ def average_error(self, p2ds, median=False):+ p3ds = self.triangulate(p2ds)+ errors = self.reprojection_error(p3ds, p2ds, mean=True)+ if median:+ return np.median(errors)+ else:+ return np.mean(errors)++ def calibrate_rows(self, all_rows, board,+ init_intrinsics=True, init_extrinsics=True, verbose=True,+ **kwargs):+ """Assumes camera sizes are set properly"""+ for rows, camera in zip(all_rows, self.cameras):+ size = camera.get_size()++ assert size is not None, \+ "Camera with name {} has no specified frame size".format(camera.get_name())++ if init_intrinsics:+ objp, imgp = board.get_all_calibration_points(rows)+ mixed = [(o, i) for (o, i) in zip(objp, imgp) if len(o) > 5]+ objp, imgp = zip(*mixed)+ matrix = cv2.initCameraMatrix2D(objp, imgp, tuple(size))+ camera.set_camera_matrix(matrix)++ for i, (row, cam) in enumerate(zip(all_rows, self.cameras)):+ all_rows[i] = board.estimate_pose_rows(cam, row)++ merged = merge_rows(all_rows)+ imgp, extra = extract_points(merged, board, min_cameras=2)++ if init_extrinsics:+ rtvecs = extract_rtvecs(merged)+ if verbose:+ pprint(get_connections(rtvecs, self.get_names()))+ rvecs, tvecs = get_initial_extrinsics(rtvecs)+ self.set_rotations(rvecs)+ self.set_translations(tvecs)++ error = self.bundle_adjust_iter(imgp, extra, verbose=verbose, **kwargs)++ return error,merged++ def get_rows_videos(self, videos, board, verbose=True):+ all_rows = []++ for cix, (cam, cam_videos) in enumerate(zip(self.cameras, videos)):+ rows_cam = []+ for vnum, vidname in enumerate(cam_videos):+ if verbose: print(vidname)+ rows = board.detect_video(vidname, prefix=vnum, progress=verbose)+ if verbose: print("{} boards detected".format(len(rows)))+ rows_cam.extend(rows)+ all_rows.append(rows_cam)++ return all_rows++ def set_camera_sizes_videos(self, videos):+ for cix, (cam, cam_videos) in enumerate(zip(self.cameras, videos)):+ rows_cam = []+ for vnum, vidname in enumerate(cam_videos):+ params = get_video_params(vidname)+ size = (params['width'], params['height'])+ cam.set_size(size)++ def calibrate_videos(self, videos, board,+ init_intrinsics=True, init_extrinsics=True, verbose=True,+ **kwargs):+ """Takes as input a list of list of video filenames, one list of each camera.+ Also takes a board which specifies what should be detected in the videos"""++ all_rows = self.get_rows_videos(videos, board, verbose=verbose)+ if init_extrinsics:+ self.set_camera_sizes_videos(videos)++ error,merged = self.calibrate_rows(all_rows, board,+ init_intrinsics=init_intrinsics,+ init_extrinsics=init_extrinsics,+ verbose=verbose, **kwargs)+ return error, merged++ def get_dicts(self):+ out = []+ for cam in self.cameras:+ out.append(cam.get_dict())+ return out++ def from_dicts(arr):+ cameras = []+ for d in arr:+ if 'fisheye' in d and d['fisheye']:+ cam = FisheyeCamera.from_dict(d)+ else:+ cam = Camera.from_dict(d)+ cameras.append(cam)+ return CameraGroup(cameras)++ def from_names(names, fisheye=False):+ cameras = []+ for name in names:+ if fisheye:+ cam = FisheyeCamera(name=name)+ else:+ cam = Camera(name=name)+ cameras.append(cam)+ return CameraGroup(cameras)++ def load_dicts(self, arr):+ for cam, d in zip(self.cameras, arr):+ cam.load_dict(d)++ def dump(self, fname):+ dicts = self.get_dicts()+ names = ['cam_{}'.format(i) for i in range(len(dicts))]+ master_dict = dict(zip(names, dicts))+ master_dict['metadata'] = self.metadata+ with open(fname, 'w') as f:+ toml.dump(master_dict, f)++ def load(fname):+ master_dict = toml.load(fname)+ keys = sorted(master_dict.keys())+ items = [master_dict[k] for k in keys if k != 'metadata']+ cgroup = CameraGroup.from_dicts(items)+ if 'metadata' in master_dict:+ cgroup.metadata = master_dict['metadata']+ return cgroup++ def resize_cameras(self, scale):+ for cam in self.cameras:+ cam.resize_camera(scale)diff --git a/freemocap/synccams.py b/freemocap/synccams.pynew file mode 100644index 00000000..56b8f931--- /dev/null+++ b/freemocap/synccams.py@@ -0,0 +1,11 @@+from freemocap.webcam import timesync+from tkinter import Tk+++def RunSync(session, timeStampData,numCamRange,camIDs):+ frameTable,timeTable,frameRate,resultsTable,plots = timesync.TimeSync(timeStampData,numCamRange,camIDs) #start the timesync process+ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no+ root = Tk()+ proceed = timesync.proceedGUI(root,resultsTable,plots) #create a GUI instance called proceed+ root.mainloop()+\ No newline at end of filediff --git a/freemocap/webcam/__init__.py b/freemocap/webcam/__init__.pynew file mode 100644index 00000000..ef4b4f30--- /dev/null+++ b/freemocap/webcam/__init__.py@@ -0,0 +1,7 @@+# -*- coding: utf-8 -*-+"""+Created on Mon Mar 1 13:39:34 2021++@author: Rontc+"""+diff --git a/freemocap/webcam/camsetup.py b/freemocap/webcam/camsetup.pynew file mode 100644index 00000000..df333ec7--- /dev/null+++ b/freemocap/webcam/camsetup.py@@ -0,0 +1,56 @@+++import threading+import cv2+import imutils++class VideoSetup(threading.Thread):+ def __init__(self, camID,parameterDictionary,rotNum):+ self.camID = camID+ self.parameterDictionary = parameterDictionary+ self.rotNum = rotNum+ threading.Thread.__init__(self)+ def run(self):+ #print("Starting " + self.previewName)+ self.record(self.parameterDictionary,self.rotNum)+ def record(self,parameterDictionary,rotNum):+ exposure = parameterDictionary.get('exposure')+ resWidth = parameterDictionary.get('resWidth')+ resHeight = parameterDictionary.get('resHeight')+ camName = "Camera" + str(self.camID)++ cv2.namedWindow(camName)+ cap = cv2.VideoCapture(self.camID,cv2.CAP_DSHOW)+ cap.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)+ cap.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)+ cap.set(cv2.CAP_PROP_EXPOSURE, exposure)+ #print('exposure',cap.get(cv2.CAP_PROP_EXPOSURE))+ #if not cap.isOpened():+ # raise RuntimeError('No camera found at input '+ str(self.camID))+ while True:+ ret1,frame1 = cap.read()+ if ret1 ==True:+ if rotNum is not None:+ frame1 = imutils.rotate_bound(frame1, angle= rotNum)+ cv2.imshow(camName,frame1)+ if cv2.waitKey(1) & 0xFF== 27:+ # == ord('q') for q+ break++ else:+ break+ cv2.destroyWindow(camName)++def RunSetup(cam_inputs,rotation_input,paramDict):+ if not cam_inputs:+ raise ValueError('Camera input list (cam_inputs) is empty')++ ulist = []++ for cam_input,cam_rotation in zip(cam_inputs,rotation_input):+ u = VideoSetup(cam_input,paramDict,cam_rotation)+ u.start()+ ulist.append(u)++ for k in ulist:+ k.join()\ No newline at end of filediff --git a/freemocap/webcam/checkcams.py b/freemocap/webcam/checkcams.pynew file mode 100644index 00000000..ed466a06--- /dev/null+++ b/freemocap/webcam/checkcams.py@@ -0,0 +1,27 @@+from tqdm import tqdm+import cv2++def TestDevice(source):+ cap = cv2.VideoCapture(source,cv2.CAP_DSHOW)+ #if cap is None or not cap.isOpened():+ #print('Warning: unable to open video source: ', source)++ if cap.isOpened():+ #print('Opened: ',source)+ #print('Exposure: '+ str(cap.get(cv2.CAP_PROP_EXPOSURE)))+ #time.sleep(3)+ cap.release()+ cv2.destroyAllWindows()+ open_cam = source+ return open_cam+ else:+ return None++def CreateAvailableCamList():+ openCamList = []+ for x in tqdm(range(20)):#range 20 right now to be safe+ openCamera = TestDevice(x)+ if openCamera is not None:+ openCamList.append(openCamera)++ return openCamList\ No newline at end of filediff --git a/freemocap/webcam/recordGUI.py b/freemocap/webcam/recordGUI.pynew file mode 100644index 00000000..4b91847f--- /dev/null+++ b/freemocap/webcam/recordGUI.py@@ -0,0 +1,253 @@+# -*- coding: utf-8 -*-+"""+Created on Wed Mar 3 12:18:46 2021++@author: Rontc+"""+import tkinter as tk+from tkinter import Tk,Label, Button, Frame, Listbox, Entry+from freemocap.webcam import checkcams+import pickle+import datetime+++class recordGUI:+ #this GUI detects how many cameras you have available and presents them in+ #a listbox, and allows you to pick a task for the cameras+ def __init__(self, master):+ self.master = master+ #Detect how many inputs you have available+ available_inputs = checkcams.CreateAvailableCamList()+ number_of_cams = len(available_inputs)+ cam_statement = "You have " + str(number_of_cams) + " cameras available at inputs " + str(available_inputs)++ self.master.title("Choose Your Cameras/Task")+ bottom_frame = Frame(self.master, height = 100)+ bottom_frame.pack(side = tk.BOTTOM)+ #---creates a listbox with all the available camera choices+ self.list_box = Listbox(self.master,selectmode = 'multiple')+ for i in available_inputs:+ self.list_box.insert(i,str(i))++ #---setting up the task radiobutton+ #creates the different options and the values associated with them+ options = [("Setup", 'setup'),+ ("Record", 'record')]++ #initializes currentTask as a string variable and defaults it to setup+ self.currentTask = tk.StringVar()+ self.currentTask.set('setup')++ #---display the detection results+ tk.Label(self.master, text = cam_statement, justify = tk.RIGHT).pack()+ tk.Label(self.master, text = 'Select Your Cameras', justify = tk.RIGHT).pack()+ self.list_box.pack()++ #---create the task radiobutton+ tk.Label(self.master,+ text="""Choose your task""",+ justify = tk.LEFT,+ padx = 20).pack()++ for option, val in options:+ tk.Radiobutton(self.master,+ text=option,+ padx = 20,+ variable= self.currentTask,+ value=val).pack(anchor=tk.W)+++ self.sub_button = Button(bottom_frame, text="Submit", command=self.Submit)+ self.sub_button.pack(side = tk.RIGHT)++++ def Submit(self):+ self.taskChoice = self.currentTask.get()+ self.selected = [int(self.list_box.get(pos)) for pos in self.list_box.curselection()]+ self.master.destroy()+++class SettingsGUI:+ def __init__(self, master,cam_inputs,parameter_dictionary,rotation_dictionary,sessionID_in,task):+ self.master = master+ self.cam_inputs = cam_inputs+ self.parameter_dictionary = parameter_dictionary+ self.rotation_dictionary = rotation_dictionary+ self.sessionID_in = sessionID_in+ self.task = task+ #check if any previous parameters exist+ if self.parameter_dictionary is not None:+ existing_parameters = True+ else:+ existing_parameters = False++ #check to see if there are any stored rotations for the cameras chosen+ rotationValues = self.RotationRetrieval(cam_inputs, rotation_dictionary)+++ #create frames to organize rotation inputs, resolution inputs, and parameter inputs+ topFrame = Frame(self.master)+ topFrame.pack(side = tk.TOP)+ rotLabelFrame = Frame(self.master)+ rotLabelFrame.pack()+ resFrame = Frame(self.master)+ resFrame.pack()+ parametersFrame = Frame(self.master)+ parametersFrame.pack()+ bottomFrame = Frame(self.master).pack(side = tk.BOTTOM)+++ taskText = "Current Task: " + self.task+ taskLabel = Label(topFrame,text = taskText, font = 'bold')+ taskLabel.pack()+ #---Rotation Options. Dynamically creates radio button options depending+ #---on how many cameras there are+ rotation_options = [("rotate 0",0),("rotate 90",90),("rotate 180",180),("rotate 270",270)]+ #radVar = tk.IntVar() #Radio Button Variable+ self.rotation_list = []+ rotationRadioButton = {}+ #creates radio buttons for each cam and stores the chosen rotation degree in a list+ for cam,rotDegree in zip(self.cam_inputs,rotationValues):+ current_rotation = tk.IntVar()+ current_rotation.set(rotDegree)+ self.rotation_list.append(current_rotation)+ tk.Label(rotLabelFrame, text = "Choose rotation for Cam " + str(cam)).pack()+ for option,degree in rotation_options:+ rotationRadioButton[cam] = tk.Radiobutton(rotLabelFrame,text=option,value = degree,variable = current_rotation ).pack()++ #---Resolution Height and Resolution Width entry+ widthLabel = Label(resFrame, text = 'Resolution Width').pack(side = 'left')+ self.resWidthEntry = Entry(resFrame)+ self.resWidthEntry.pack(side = 'left')+ heightLabel = Label(resFrame, text = 'Resolution Height').pack(side = 'left')+ self.resHeightEntry = Entry(resFrame)+ self.resHeightEntry.pack(side = 'left')++ #---Exposure,FPS, and codec entry+ exposureLabel = Label(parametersFrame, text = 'Exposure').pack(side= 'left')+ self.exposureEntry = Entry(parametersFrame)+ self.exposureEntry.pack(side = 'left')+ FPSLabel = Label(parametersFrame, text = 'FPS').pack(side= 'left')+ self. FPSEntry = Entry(parametersFrame)+ self.FPSEntry.pack( side = 'left')+ codecLabel = Label(parametersFrame, text = 'codec').pack(side = 'left')+ self.codecEntry = Entry(parametersFrame)+ self.codecEntry.pack(side = 'left')++ #---SessionID entry- default is the date/time+ if self.task == 'record':+ sessionText = 'Change SessionID if desired'+ sessionLabel = Label(parametersFrame, text = sessionText).pack()+ self.sessionIDEntry = Entry(parametersFrame)+ self.sessionIDEntry.pack()+ self.sessionIDEntry.insert(0,self.sessionID_in)++ #---If previous parameters were recieved, insert them into the entry boxes+ if existing_parameters == True:++ self.exposureEntry.insert(0,parameter_dictionary['exposure'])+ self.resWidthEntry.insert(0,parameter_dictionary['resWidth'])+ self.resHeightEntry.insert(0,parameter_dictionary['resHeight'])+ self.codecEntry.insert(0,parameter_dictionary['codec'])+ self.FPSEntry.insert(0,parameter_dictionary['framerate'])++ #---Submit button to hit when things are finalized+ self.sub_button = Button(bottomFrame, text="Submit", command= self.Submit)+ self.sub_button.pack(side = tk.BOTTOM)+++ def Submit(self):+ #get the final entries for all parameters+ #get the rotations for each camera and save them as values of a dictionary+ self.rotation_output = {}+ for count,cam in enumerate(self.cam_inputs):+ self.rotation_output[cam] = self.rotation_list[count].get()++ #built a parameter dictionary by .get()-ing the values from each entry+ self.paramDict = {'exposure':int(self.exposureEntry.get()),"resWidth":int(self.resWidthEntry.get()),"resHeight":int(self.resHeightEntry.get()),+ 'framerate':int(self.FPSEntry.get()),'codec':str(self.codecEntry.get())}+ #spit out the sessionID+ if self.task =='record':+ self.sessionID_out = self.sessionIDEntry.get()++ self.master.destroy()+ def RotationRetrieval(self,cam_inputs,rotation_dictionary):+ #takes the input cameras and compares them to the dictionary of+ #rotation values from the last session. For any matching cameras,+ #grab the rotation values. Otherwise, the rotation value is None+ rotationValues = []+ for cam in cam_inputs:+ if cam in rotation_dictionary.keys():+ rotationValues.append(rotation_dictionary[cam])+ else:+ rotationValues.append(0)+ return rotationValues++++class ProceedToRecordGUI():+ def __init__(self, master,sessionID_in):+ self.master = master+ self.sessionID_in = sessionID_in+ self.continueToRecording = ''+ self.sessionID_out = ''++ sessionText = "Proceed to recording with this sessionID?"+ sessionLabel = Label(master,text = sessionText)+ sessionLabel.pack(side = 'top')++ self.sessionIDEntry = Entry(master)+ self.sessionIDEntry.pack()+ self.sessionIDEntry.insert(0,self.sessionID_in)++ #master.title("Proceed to Recording?")+ self.proceed_button = Button(text="Proceed", command=self.destroy)+ self.stop_button = Button(text="Quit", command= self.stop)+ self.stop_button.pack(side = tk.RIGHT)+ self.proceed_button.pack(side = tk.RIGHT)++ def stop(self):+ self.master.destroy()++ def destroy(self):+ self.continueToRecording = True+ self.sessionID_out = self.sessionIDEntry.get()+ self.master.destroy()++def RunGUI(sessionID_in, rotation_entry, parameter_entry):++ #---Get the camera inputs and the task+ root = tk.Tk()+ camera_choice = recordGUI(root)+ root.mainloop()++ cam_inputs = camera_choice.selected+ if not cam_inputs:+ raise ValueError('No camera inputs selected')+ task = camera_choice.taskChoice+++ #---Get all the necessary parameters+ root = tk.Tk()+ recording_settings = SettingsGUI(root,cam_inputs,parameter_entry,rotation_entry,sessionID_in,task)+ root.mainloop()++ #---Create the parameters we'll need to run the records+ #rotation_input = list(test.rotation_output.values())++ paramDict = recording_settings.paramDict++ if task == 'record':+ sessionID = recording_settings.sessionID_out+ else:+ sessionID = None++ return task, cam_inputs, recording_settings.rotation_output, paramDict, sessionID++def RunProceedtoRecordGUI(sessionID_in):+ root = tk.Tk()+ goToRecording = ProceedToRecordGUI(root,sessionID_in)+ root.mainloop()++ return goToRecording.continueToRecording,goToRecording.sessionID_outdiff --git a/freemocap/webcam/runrecordingprocess.py b/freemocap/webcam/runrecordingprocess.pynew file mode 100644index 00000000..a30d2bd1--- /dev/null+++ b/freemocap/webcam/runrecordingprocess.py@@ -0,0 +1,80 @@+from freemocap import recordingconfig+from freemocap.webcam import startcamrecording,timesync,videotrim+from pathlib import Path+import time+import pickle+import pandas as pd+import numpy as np+from tkinter import Tk,Label, Button, Frame, Listbox, Entry+++def RunCams(session,camInputs,parameterDictionary,rotationInput):++++ #%% Setting up recordings+ beginTime = time.time()+ numCams = len(camInputs) #number of cameras+ session.numCams = numCams+ numCamRange = range(numCams) #a range for the number of cameras that we have+ videoNames = []+ camIDs = []+ for x in numCamRange: #create names for each of the initial untrimmed videos+ singleCamID = 'Cam{}'.format(x+1)+ camIDs.append(singleCamID) #creates IDs for each camera based on the number of cameras entered+ singleVidName = 'raw_cam{}.mp4'.format(x+1)+ videoNames.append(singleVidName)++ #%% Starting the thread recordings for each camera+ threads = []+ for n in numCamRange: #starts recording video, opens threads for each camera+ camRecordings = startcamrecording.CamRecordingThread(camIDs[n],camInputs[n],videoNames[n],session.rawVidPath,beginTime,parameterDictionary)+ camRecordings.start()++ threads.append(camRecordings)++ for camRecordings in threads:+ camRecordings.join() #make sure that one thread ending doesn't immediately end all the others (before they can dump data in a pickle file)++ print('finished recordings')++ timeStampList = []++ for e in numCamRange: #open the saved pickle file for each camera, and add the timestamps to the dataList list+ with open(camIDs[e], 'rb') as f:+ camTimeList = pickle.load(f)+ timeStampList.append(camTimeList)++ timeDictionary = {}++ id_and_time = zip(camIDs,timeStampList)++ for cam,data in id_and_time:+ timeDictionary[cam] = np.array(data) #create a dictionary that holds the timestamps for each camera+++ df = pd.DataFrame.from_dict(timeDictionary, orient = 'index') #create a data frame from this dictionary+ dfT = df.transpose()+ csvName = session.sessionID + '.csv' #create our csv filename+ csvPath = session.sessionPath/csvName+ dfT.to_csv(csvPath) #turn dataframe into a CSV+++ frameTable,timeTable,frameRate,resultsTable,plots = timesync.TimeSync(dfT,numCamRange,camIDs) #start the timesync process+ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no+ root = Tk()+ proceed = timesync.proceedGUI(root,resultsTable,plots) #create a GUI instance called proceed+ root.mainloop()+++ if proceed.proceed == True: #check if the proceed attribute is set to True+ print()+ print('Starting editing')+ #start editing the videos+ videotrim.VideoTrim(session,videoNames,frameTable,parameterDictionary,rotationInput,numCamRange)+ videotrim.createCalibrationVideos(session,60,parameterDictionary)+ print('all done')++ return proceed.proceed++\ No newline at end of filediff --git a/freemocap/webcam/startcamrecording.py b/freemocap/webcam/startcamrecording.pynew file mode 100644index 00000000..9a433d08--- /dev/null+++ b/freemocap/webcam/startcamrecording.py@@ -0,0 +1,80 @@+import threading+import cv2+import time+import pickle++class CamRecordingThread(threading.Thread):+ def __init__(self,camID,camInput,videoName,rawVidPath,beginTime,parameterDictionary):+ threading.Thread.__init__(self)+ self.camID = camID+ self.camInput = camInput+ self.videoName = videoName+ self.rawVidPath = rawVidPath+ self.beginTime = beginTime+ self.parameterDictionary = parameterDictionary++ def run(self):+ print("Starting " + self.camID)+ self.timeStamps = CamRecording(self.camID, self.camInput, self.videoName,self.rawVidPath,self.beginTime,self.parameterDictionary)+ def getStamps(self):+ return self.timeStamps+#the recording function that each threaded camera object runs+def CamRecording(camID, camInput, videoName,rawVidPath,beginTime,parameterDictionary):+ #the flag is triggered when the user shuts down one webcam to shut down the rest.+ #normally I'd try to avoid global variables, but in this case it's+ #necessary, since each webcam runs as it's own object.+ global flag+ flag = False++ cv2.namedWindow(camID) #name the preview window for the camera its showing+ cam = cv2.VideoCapture(camInput,cv2.CAP_DSHOW) #create the video capture object+ #if not cam.isOpened():+ # raise RuntimeError('No camera found at input '+ str(camID))+ #pulling out all the dictionary paramters+ exposure = parameterDictionary.get('exposure')+ resWidth = parameterDictionary.get('resWidth')+ resHeight = parameterDictionary.get('resHeight')+ framerate = parameterDictionary.get('framerate')+ codec = parameterDictionary.get('codec')++ cam.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)+ cam.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)+ cam.set(cv2.CAP_PROP_EXPOSURE, exposure)+ fourcc = cv2.VideoWriter_fourcc(*codec)+ #rawPath = filepath/'RawVideos' #creating a RawVideos folder+ #rawPath.mkdir(parents = True, exist_ok = True)+ width = cam.get(cv2.CAP_PROP_FRAME_WIDTH)+ height = cam.get(cv2.CAP_PROP_FRAME_HEIGHT)+ print('width:',width, 'height:',height)+ saveRawVidPath = str(rawVidPath/videoName) #create a save path for each video to the RawVideos folders+ out = cv2.VideoWriter(saveRawVidPath,fourcc, framerate, (resWidth,resHeight))+ timeStamps = [] #holds the timestamps++ if cam.isOpened():+ success, frame = cam.read()+ else:+ success = False++ while success: #while the camera is opened, record the data until the escape button is hit+ if flag: #when the flag is triggered, stop recording and dump the data+ with open(camID, 'wb') as f:+ pickle.dump(timeStamps, f)+ break+ success, frame = cam.read()++ cv2.imshow(camID, frame)+ frame_sized = cv2.resize(frame,(resWidth,resHeight))+ #frame_sized = frame+ out.write(frame_sized)+ timeStamps.append(time.time()-beginTime) #add each timestamp to the list++ key = cv2.waitKey(20)+ if key == 27: # exit on ESC+ flag = True #set flag to true to shut down all other webcams+ with open(camID, 'wb') as f:+ pickle.dump(timeStamps, f) #dump the data+ break+ cv2.destroyWindow(camID)+ return timeStamps++#this is how we sync our time frames, based on our recorded timestamps\ No newline at end of filediff --git a/freemocap/webcam/timesync.py b/freemocap/webcam/timesync.pynew file mode 100644index 00000000..814fe391--- /dev/null+++ b/freemocap/webcam/timesync.py@@ -0,0 +1,187 @@+import numpy as np+import pandas as pd+import sys+from matplotlib.figure import Figure+from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg+import tkinter as tk+from tkinter import Tk,Label, Button, Frame++def TimeSync(timeStampData,numCamRange,camNames):++ def CloseNeighb(camera,point):+ closestPoint = (np.abs(camera - point)).argmin()+ return closestPoint+++ #this section auto-finds the start and end points for our master timeline to the nearest second+ masterTimelineBegin = np.ceil(max(timeStampData.iloc[0]))+ lastPoints = []+ for name in camNames:+ cameraLastPoint = timeStampData[name][timeStampData[name].last_valid_index()] #find the last non-nan value in each camera+ lastPoints.append(cameraLastPoint)++ masterTimelineEnd = np.floor(min(lastPoints)) #find where the fastest camera ended, round down, use that as the end point+++ #print('end',endList)+ print('intervals:',masterTimelineBegin,masterTimelineEnd)+++ totalFrameIntvl= [] #list for storing frame rates for each camera+++ n = 0; #counter for going through camera names+ for x in numCamRange:+ currentCam = timeStampData[camNames[x]]+ #finds the closest point in each camera to the start and end points of our timeline+ currentCamStart = CloseNeighb(currentCam,masterTimelineBegin)+ currentCamEnd = CloseNeighb(currentCam,masterTimelineEnd)+ print("using frames:", currentCamStart,"-",currentCamEnd,"for",camNames[n])+ n +=1+ currentCamTimeline = currentCam[currentCamStart:currentCamEnd] #grab the times from start to finish for each camera+ currentCamFrameInterval = np.mean(np.diff(currentCamTimeline)) #calculate the interval between each frame and take the mean+ totalFrameIntvl.append(currentCamFrameInterval) #add interval to list+ #print(camNames[x],currentCamStart,currentCamEnd)+ totalAverageIntvl = np.mean(totalFrameIntvl) #find the total average interval across all cameras+ masterTimeline = np.arange(masterTimelineBegin,masterTimelineEnd,totalAverageIntvl) #build a master timeline with the average interval++ #now we start the syncing process++ frameList = masterTimeline #start a list of frames, with the first row being our master timeline+ timeList = masterTimeline #start a list of timestamps,with the first row being our master timeline++ delNum= []; #stores number of deleted frames/camera+ bufNum= []; #stores number of buffered frames/camera+ bufPercentList = []; #stores percentage of deleted frames/camera+ delPercentList = []; #stores percentage of buffered frames/camera++ count = 0 #Keeps track of what frame we're on (I think?)+ n = 0; #I don't remember why I did this but I'm sure I'll figure it out later++ for y in numCamRange:+ thisCam = timeStampData[camNames[y]]+ camTimes = []; #stored times+ camFrames =[]; #stored frames++ count += 1+ #--------------Adjust each camera to start at the first point in the master timeline+ beginFrame = CloseNeighb(thisCam,masterTimeline[0])+ timeDif = masterTimeline[0] - thisCam[beginFrame]+ thisCam = thisCam + timeDif++ for z in masterTimeline: #for each point in the master timeline+ closestFrame = CloseNeighb(thisCam, z) #find the closest frame in this camera to each point of the master timeline+ camFrames.append(closestFrame) #add that frame to the list+ camTimes.append(thisCam[closestFrame]) #find the time corresponding to this frame++ print("starting detection:",camNames[n]) #now to start finding deleted/buffered slides+ frameList = np.column_stack((frameList,camFrames)) #update our framelist with our new frames+ timeList = np.column_stack((timeList,camTimes)) #update our timelist+++ #start counters for the number of buffered and deleted slides+ bufCount = 0;+ delCount = 0;++ for i in range(0,len(camFrames)-1):+ distanceBetweenFrames = camFrames[i+1] - camFrames[i] #find the distance between adjacent frames+ if distanceBetweenFrames == 1: #these frames are consecutive, do nothing+ None+ elif distanceBetweenFrames == 0: #we have a buffered slide+ bufCount += 1+ #this section looks at our two timepoints, and finds the which point on the master timeline is closest+ frame1 = abs(masterTimeline[i]-camTimes[i])+ frame2 = abs(masterTimeline[i+1]-camTimes[i])+ #finds which frame is closest, and sets the other frame as a buffer (indicated by the -1)+ if frame1>frame2:+ frameList[i,count] = -1+ timeList[i,count] = -1+ else:+ frameList[i+1,count] = -1;+ timeList[i+1,count] = -1;+ elif distanceBetweenFrames > 1: #deletion+ delCount += 1+ else:++ print("something else happened")+ #update and calculate our percentages and numbers for buffers/deletions+ delNum.append(round(delCount,1))+ bufNum.append(round(bufCount,1))+ #print(bufCount,len(frameList))+ bufPercent = (bufCount/len(frameList))*100+ delPercent = (delCount/len(frameList))*100+ bufPercentList.append(round(bufPercent,2))+ delPercentList.append(round(delPercent,2))++ #print(delNum,bufNum,bufPercentList,delPercentList)+ #print("deleted frames:",delCount)+ #print("buffered frames:",bufCount)+ n +=1++ #create our data frame for both times and frames+ frameTable = pd.DataFrame(frameList)+ columnNames = ['Master Timeline'] + camNames+ frameTable.columns = columnNames+ timeTable = pd.DataFrame(timeList)+ timeTable.columns = columnNames+ totalFrameRate = [round(1/intvl,1) for intvl in totalFrameIntvl]+ frameRate = 1/totalAverageIntvl #calculates our framerate+ results = {'Cam':camNames,'#Del':delNum,'%Del':delPercentList,'#Buf':bufNum,'%Buf':bufPercentList,'FPS':totalFrameRate}+ resultTable = pd.DataFrame(results,columns = ['Cam','#Del','%Del','#Buf','%Buf','FPS'])++ #============================================== plot data+ differenceFrame = timeStampData.diff(axis = 0)+ fpsFrame = differenceFrame.pow(-1)+ fig = Figure(figsize = [10,10])+ fig.patch.set_facecolor('#F0F0F0')+ a = fig.add_subplot(221)+ a.set_xlabel('Frame')+ a.set_ylabel('Time(s)')+ timeStampData.plot(ax = a, title = 'Camera timestamps')+ b = fig.add_subplot(222)+ b.set_xlabel('Frame')+ b.set_ylabel('Frame duration (s)')+ differenceFrame.plot(ax = b, marker = '.', linestyle = 'none',title = 'Camera frame duration')+ c = fig.add_subplot(223)+ c.set_xlabel('Time (s)')+ differenceFrame.plot.hist(ax = c, bins=6,alpha = .5, xlim = [0,.1], title = 'Frame interval distribution (s)')+ d = fig.add_subplot(224)+ d.set_xlabel('Frames per second')+ fpsFrame.plot.hist(ax = d, bins=6,alpha = .5, xlim = [0,40], title = 'Frames per second distribution')+ return frameTable,timeTable,frameRate,resultTable,fig+#function to trim our videos++class proceedGUI:+ def __init__(self, master, results, figure):+ self.master = master+ self.results = results+ self.figure = figure+ self.proceed = ''+ master.title("Choose File Path")+++ bottom_frame = Frame(self.master, height = 1000)+ bottom_frame.pack(side = tk.LEFT)+ self.label = Label(bottom_frame, text= results.to_string(index = False))+ self.label.pack(side = tk.TOP)+ self.go_button = Button(bottom_frame, text="Proceed", command=self.destroy)+ self.stop_button = Button(bottom_frame, text="Quit", command= self.stop)++ self.canvas = FigureCanvasTkAgg(self.figure, master=self.master)+ self.canvas.draw()+ self.canvas.get_tk_widget().pack(side=tk.LEFT, fill="both", expand=1)++ self.stop_button.pack(side = tk.RIGHT)+ self.go_button.pack(side = tk.RIGHT)++++++ def stop(self):+ self.master.destroy()+ sys.exit("Quitting Program")++ def destroy(self):+ self.proceed = True+ self.master.destroy()\ No newline at end of filediff --git a/freemocap/webcam/videotrim.py b/freemocap/webcam/videotrim.pynew file mode 100644index 00000000..4f4e8924--- /dev/null+++ b/freemocap/webcam/videotrim.py@@ -0,0 +1,80 @@+from rich.progress import track+import cv2+import imutils+import numpy as np+import os++def VideoTrim(session,vidList,ft,parameterDictionary,rotationState,numCamRange):+ camList = list(ft.columns[1:len(vidList)+1]) #grab the camera identifiers from the data frame+ resWidth = parameterDictionary.get('resWidth')+ resHeight = parameterDictionary.get('resHeight')+ framerate = parameterDictionary.get('framerate')+ trueWidth = resWidth #creating true resWidth and resHeight in case we need to rotate our images+ trueHeight = resHeight+ codec = parameterDictionary.get('codec')++ for vid,cam,camNum in zip(vidList,camList,numCamRange): #iterate in parallel through camera identifiers and matching videos+ print('Editing '+cam+' from ' +vid)+ cap = cv2.VideoCapture(str(session.rawVidPath/vid)) #initialize OpenCV capture+ frameTable = ft[cam] #grab the frames needed for that camera+ success, image = cap.read() #start reading frames+ fourcc = cv2.VideoWriter_fourcc(*codec)+ saveName = session.sessionID +'_synced_' + cam + '.mp4'+++ saveSyncedVidPath = str(session.syncedVidPath/saveName) #create an output path for the function+ if rotationState[camNum] == 90 or rotationState[camNum] == 270:+ #if we are rotating to these angles, we need to temporarily swap our resWidth and resHeight+ tempHeight = resHeight+ resHeight = resWidth+ resWidth = tempHeight++ out = cv2.VideoWriter(saveSyncedVidPath, fourcc, framerate, (resWidth,resHeight)) #change resolution as needed+ for frame in track(frameTable): #start looking through the frames we need+ if frame == -1: #this is a buffer frame+ blankFrame = np.zeros_like(image) #create a blank frame+ if rotationState[camNum] is not None:+ blankFrame = imutils.rotate_bound(blankFrame, angle= rotationState[camNum])+ image = cv2.resize(image,(resWidth,resHeight))+ out.write(blankFrame) #write that frame to the video+ else:+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need+ success, image = cap.read()+ if rotationState[camNum] is not None:+ image = imutils.rotate_bound(image, angle= rotationState[camNum])+ image = cv2.resize(image,(resWidth,resHeight))+ out.write(image)+ resWidth = trueWidth+ resHeight = trueHeight+ cap.release()+ out.release()+ print('Saved '+ saveSyncedVidPath)+ print()++def createCalibrationVideos(session,calVideoFrameLength,parameterDictionary):+ vidList = os.listdir(session.syncedVidPath)+ framelist = list(range(calVideoFrameLength))+ codec = parameterDictionary.get('codec')+ for count,vid in enumerate(vidList,start=1):+ cam_name = "Cam{}".format(count)+ cap = cv2.VideoCapture(str(session.syncedVidPath/vid))+ fourcc = cv2.VideoWriter_fourcc(*codec)++ #grab resolution parameters from the videos+ resWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))+ resHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))+ framerate = int(cap.get(cv2.CAP_PROP_FPS))++ saveName = session.sessionID +'_trimmed_' + cam_name + '.mp4' #create a name for the trimmed video+ saveCalVidPath = str(session.calVidPath/saveName) #create an output path for the function++ success, image = cap.read() #start reading frames++ out = cv2.VideoWriter(saveCalVidPath, fourcc, framerate, (resWidth,resHeight))+ print('Trimming ' + cam_name)+ for frame in track(framelist):+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need+ success, image = cap.read()+ out.write(image)+ cap.release()+ out.release()\ No newline at end of filediff --git a/freemocap/webcam/webcam.py b/freemocap/webcam/webcam.pynew file mode 100644index 00000000..31ced8fe--- /dev/null+++ b/freemocap/webcam/webcam.py@@ -0,0 +1,632 @@+# -*- coding: utf-8 -*-+"""+Created on Mon Nov 30 12:48:14 2020++@author: Rontc+"""++import cv2+import threading+import pickle+import time+import numpy as np+import pandas as pd+import tkinter as tk+from tkinter import Tk,Label, Button, Frame+#import tkMessageBox+from tqdm import tqdm+import imutils+import sys+from matplotlib.figure import Figure+from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg+from omc import recordingconfig+from pathlib import Path++global beginTime+++#create a camera object that can be threaded+class CamThread(threading.Thread):+ def __init__(self,camID,camInput,videoName,rawVidPath,beginTime,parameterDictionary):+ threading.Thread.__init__(self)+ self.camID = camID+ self.camInput = camInput+ self.videoName = videoName+ self.rawVidPath = rawVidPath+ self.beginTime = beginTime+ self.parameterDictionary = parameterDictionary++ def run(self):+ print("Starting " + self.camID)+ self.timeStamps = CamPreview(self.camID, self.camInput, self.videoName,self.rawVidPath,self.beginTime,self.parameterDictionary)+ def getStamps(self):+ return self.timeStamps+#the recording function that each threaded camera object runs+def CamPreview(camID, camInput, videoName,rawVidPath,beginTime,parameterDictionary):+ #the flag is triggered when the user shuts down one webcam to shut down the rest.+ #normally I'd try to avoid global variables, but in this case it's+ #necessary, since each webcam runs as it's own object.+ global flag+ flag = False++ cv2.namedWindow(camID) #name the preview window for the camera its showing+ cam = cv2.VideoCapture(camInput,cv2.CAP_DSHOW) #create the video capture object+ #if not cam.isOpened():+ # raise RuntimeError('No camera found at input '+ str(camID))+ #pulling out all the dictionary paramters+ exposure = parameterDictionary.get('exposure')+ resWidth = parameterDictionary.get('resWidth')+ resHeight = parameterDictionary.get('resHeight')+ framerate = parameterDictionary.get('framerate')+ codec = parameterDictionary.get('codec')++ cam.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)+ cam.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)+ cam.set(cv2.CAP_PROP_EXPOSURE, exposure)+ fourcc = cv2.VideoWriter_fourcc(*codec)+ #rawPath = filepath/'RawVideos' #creating a RawVideos folder+ #rawPath.mkdir(parents = True, exist_ok = True)+ width = cam.get(cv2.CAP_PROP_FRAME_WIDTH)+ height = cam.get(cv2.CAP_PROP_FRAME_HEIGHT)+ print('width:',width, 'height:',height)+ saveRawVidPath = str(rawVidPath/videoName) #create a save path for each video to the RawVideos folders+ out = cv2.VideoWriter(saveRawVidPath,fourcc, framerate, (resWidth,resHeight))+ timeStamps = [] #holds the timestamps++ if cam.isOpened():+ success, frame = cam.read()+ else:+ success = False++ while success: #while the camera is opened, record the data until the escape button is hit+ if flag: #when the flag is triggered, stop recording and dump the data+ with open(camID, 'wb') as f:+ pickle.dump(timeStamps, f)+ break+ success, frame = cam.read()++ cv2.imshow(camID, frame)+ frame_sized = cv2.resize(frame,(resWidth,resHeight))+ #frame_sized = frame+ out.write(frame_sized)+ timeStamps.append(time.time()-beginTime) #add each timestamp to the list++ key = cv2.waitKey(20)+ if key == 27: # exit on ESC+ flag = True #set flag to true to shut down all other webcams+ with open(camID, 'wb') as f:+ pickle.dump(timeStamps, f) #dump the data+ break+ cv2.destroyWindow(camID)+ return timeStamps++#this is how we sync our time frames, based on our recorded timestamps+class proceedGUI:+ def __init__(self, master, results, figure):+ self.master = master+ self.results = results+ self.figure = figure+ master.title("Choose File Path")+++ bottom_frame = Frame(self.master, height = 1000)+ bottom_frame.pack(side = tk.LEFT)+ self.label = Label(bottom_frame, text= results.to_string(index = False))+ self.label.pack(side = tk.TOP)+ self.go_button = Button(bottom_frame, text="Proceed", command=self.destroy)+ self.stop_button = Button(bottom_frame, text="Quit", command= self.stop)++ self.canvas = FigureCanvasTkAgg(self.figure, master=self.master)+ self.canvas.draw()+ self.canvas.get_tk_widget().pack(side=tk.LEFT, fill="both", expand=1)++ self.stop_button.pack(side = tk.RIGHT)+ self.go_button.pack(side = tk.RIGHT)++++++ def stop(self):+ self.master.destroy()+ sys.exit("Quitting Program")++ def destroy(self):+ self.master.destroy()++def TimeSync(df,numCamRange,camNames):++ def CloseNeighb(camera,point):+ closestPoint = (np.abs(camera - point)).argmin()+ return closestPoint++ newFrame = df #read our CSV file++++ #this section auto-finds the start and end points for our master timeline to the nearest second+ masterTimelineBegin = np.ceil(max(newFrame.iloc[0]))+ lastPoints = []+ for name in camNames:+ cameraLastPoint = newFrame[name][newFrame[name].last_valid_index()] #find the last non-nan value in each camera+ lastPoints.append(cameraLastPoint)++ masterTimelineEnd = np.floor(min(lastPoints)) #find where the fastest camera ended, round down, use that as the end point+++ #print('end',endList)+ print('intervals:',masterTimelineBegin,masterTimelineEnd)+++ totalFrameIntvl= [] #list for storing frame rates for each camera+++ n = 0; #counter for going through camera names+ for x in numCamRange:+ currentCam = newFrame[camNames[x]]+ #finds the closest point in each camera to the start and end points of our timeline+ currentCamStart = CloseNeighb(currentCam,masterTimelineBegin)+ currentCamEnd = CloseNeighb(currentCam,masterTimelineEnd)+ print("using frames:", currentCamStart,"-",currentCamEnd,"for",camNames[n])+ n +=1+ currentCamTimeline = currentCam[currentCamStart:currentCamEnd] #grab the times from start to finish for each camera+ currentCamFrameInterval = np.mean(np.diff(currentCamTimeline)) #calculate the interval between each frame and take the mean+ totalFrameIntvl.append(currentCamFrameInterval) #add interval to list+ #print(camNames[x],currentCamStart,currentCamEnd)+ totalAverageIntvl = np.mean(totalFrameIntvl) #find the total average interval across all cameras+ masterTimeline = np.arange(masterTimelineBegin,masterTimelineEnd,totalAverageIntvl) #build a master timeline with the average interval++ #now we start the syncing process++ frameList = masterTimeline #start a list of frames, with the first row being our master timeline+ timeList = masterTimeline #start a list of timestamps,with the first row being our master timeline++ delNum= []; #stores number of deleted frames/camera+ bufNum= []; #stores number of buffered frames/camera+ bufPercentList = []; #stores percentage of deleted frames/camera+ delPercentList = []; #stores percentage of buffered frames/camera++ count = 0 #Keeps track of what frame we're on (I think?)+ n = 0; #I don't remember why I did this but I'm sure I'll figure it out later++ for y in numCamRange:+ thisCam = newFrame[camNames[y]]+ camTimes = []; #stored times+ camFrames =[]; #stored frames++ count += 1+ #--------------Adjust each camera to start at the first point in the master timeline+ beginFrame = CloseNeighb(thisCam,masterTimeline[0])+ timeDif = masterTimeline[0] - thisCam[beginFrame]+ thisCam = thisCam + timeDif++ for z in masterTimeline: #for each point in the master timeline+ closestFrame = CloseNeighb(thisCam, z) #find the closest frame in this camera to each point of the master timeline+ camFrames.append(closestFrame) #add that frame to the list+ camTimes.append(thisCam[closestFrame]) #find the time corresponding to this frame++ print("starting detection:",camNames[n]) #now to start finding deleted/buffered slides+ frameList = np.column_stack((frameList,camFrames)) #update our framelist with our new frames+ timeList = np.column_stack((timeList,camTimes)) #update our timelist+++ #start counters for the number of buffered and deleted slides+ bufCount = 0;+ delCount = 0;++ for i in range(0,len(camFrames)-1):+ distanceBetweenFrames = camFrames[i+1] - camFrames[i] #find the distance between adjacent frames+ if distanceBetweenFrames == 1: #these frames are consecutive, do nothing+ None+ elif distanceBetweenFrames == 0: #we have a buffered slide+ bufCount += 1+ #this section looks at our two timepoints, and finds the which point on the master timeline is closest+ frame1 = abs(masterTimeline[i]-camTimes[i])+ frame2 = abs(masterTimeline[i+1]-camTimes[i])+ #finds which frame is closest, and sets the other frame as a buffer (indicated by the -1)+ if frame1>frame2:+ frameList[i,count] = -1+ timeList[i,count] = -1+ else:+ frameList[i+1,count] = -1;+ timeList[i+1,count] = -1;+ elif distanceBetweenFrames > 1: #deletion+ delCount += 1+ else:++ print("something else happened")+ #update and calculate our percentages and numbers for buffers/deletions+ delNum.append(round(delCount,1))+ bufNum.append(round(bufCount,1))+ #print(bufCount,len(frameList))+ bufPercent = (bufCount/len(frameList))*100+ delPercent = (delCount/len(frameList))*100+ bufPercentList.append(round(bufPercent,2))+ delPercentList.append(round(delPercent,2))++ #print(delNum,bufNum,bufPercentList,delPercentList)+ #print("deleted frames:",delCount)+ #print("buffered frames:",bufCount)+ n +=1++ #create our data frame for both times and frames+ frameTable = pd.DataFrame(frameList)+ columnNames = ['Master Timeline'] + camNames+ frameTable.columns = columnNames+ timeTable = pd.DataFrame(timeList)+ timeTable.columns = columnNames+ totalFrameRate = [round(1/intvl,1) for intvl in totalFrameIntvl]+ frameRate = 1/totalAverageIntvl #calculates our framerate+ results = {'Cam':camNames,'#Del':delNum,'%Del':delPercentList,'#Buf':bufNum,'%Buf':bufPercentList,'FPS':totalFrameRate}+ resultTable = pd.DataFrame(results,columns = ['Cam','#Del','%Del','#Buf','%Buf','FPS'])++ #============================================== plot data+ differenceFrame = newFrame.diff(axis = 0)+ fpsFrame = differenceFrame.pow(-1)+ fig = Figure(figsize = [10,10])+ fig.patch.set_facecolor('#F0F0F0')+ a = fig.add_subplot(221)+ a.set_xlabel('Frame')+ a.set_ylabel('Time(s)')+ newFrame.plot(ax = a, title = 'Camera timestamps')+ b = fig.add_subplot(222)+ b.set_xlabel('Frame')+ b.set_ylabel('Frame duration (s)')+ differenceFrame.plot(ax = b, marker = '.', linestyle = 'none',title = 'Camera frame duration')+ c = fig.add_subplot(223)+ c.set_xlabel('Time (s)')+ differenceFrame.plot.hist(ax = c, bins=6,alpha = .5, xlim = [0,.1], title = 'Frame interval distribution (s)')+ d = fig.add_subplot(224)+ d.set_xlabel('Frames per second')+ fpsFrame.plot.hist(ax = d, bins=6,alpha = .5, xlim = [0,40], title = 'Frames per second distribution')+ return frameTable,timeTable,frameRate,resultTable,fig+#function to trim our videos+++def VideoEdit(rawVidPath,syncedVidPath, vidList,sessionName,ft,parameterDictionary,rotationState,numCamRange):+ camList = list(ft.columns[1:len(vidList)+1]) #grab the camera identifiers from the data frame+ resWidth = parameterDictionary.get('resWidth')+ resHeight = parameterDictionary.get('resHeight')+ framerate = parameterDictionary.get('framerate')+ trueWidth = resWidth+ trueHeight = resHeight+ codec = parameterDictionary.get('codec')+ for vid,cam,camNum in zip(vidList,camList,numCamRange): #iterate in parallel through camera identifiers and matching videos+ print('Editing '+cam+' from ' +vid)+ #print(cam+'_'+out_path)+ #rawPath = filepath/'RawVideos'+ cap = cv2.VideoCapture(str(rawVidPath/vid)) #initialize OpenCV capture+ frameTable = ft[cam] #grab the frames needed for that camera+ success, image = cap.read() #start reading frames+ fourcc = cv2.VideoWriter_fourcc(*codec)+ saveName = sessionName +'_synced_' + cam + '.mp4'+ #syncedPath = filepath/'SyncedVideos'+ #syncedPath.mkdir(parents = True, exist_ok = True)+++ saveSyncedVidPath = str(syncedVidPath/saveName) #create an output path for the function+ if rotationState[camNum] == 90 or rotationState[camNum] == 270:+ tempHeight = resHeight+ resHeight = resWidth+ resWidth = tempHeight+ #print(rotationState[camNum], resHeight, resWidth)++ out = cv2.VideoWriter(saveSyncedVidPath, fourcc, framerate, (resWidth,resHeight)) #change resolution as needed+ for frame in tqdm(frameTable, leave = True): #start looking through the frames we need+ if frame == -1: #this is a buffer frame+ blankFrame = np.zeros_like(image) #create a blank frame+ if rotationState[camNum] is not None:+ blankFrame = imutils.rotate_bound(blankFrame, angle= rotationState[camNum])+ #image = cv2.rotate(image,rotateCode = rotationState[camNum])+ image = cv2.resize(image,(resWidth,resHeight))+ out.write(blankFrame) #write that frame to the video+ else:+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame) #set the video to the frame that we need+ success, image = cap.read()+ if rotationState[camNum] is not None:+ image = imutils.rotate_bound(image, angle= rotationState[camNum])+ #image = cv2.rotate(image,rotateCode = rotationState[camNum])+ image = cv2.resize(image,(resWidth,resHeight))+ out.write(image)+ resWidth = trueWidth+ resHeight = trueHeight+ cap.release()+ out.release()+ print('Saved '+ saveSyncedVidPath)+ print()+++#function to run all these above functions together+def RunCams(camInputs,config_yaml_path,sessionName,parameterDictionary,rotationInput):+ csvName = sessionName + '.csv' #create our csv filename++ config_settings = recordingconfig.load_config_yaml(config_yaml_path)+ rawVidPath = Path(config_settings['Paths']['rawVidPath'])+ syncedVidPath = Path(config_settings['Paths']['syncedVidPath'])+++ beginTime = time.time()+ numCams = len(camInputs) #number of cameras+ numCamRange = range(numCams) #a range for the number of cameras that we have+ videoNames = []+ camIDs = []+ for x in numCamRange: #create names for each of the initial untrimmed videos+ singleCamID = 'Cam{}'.format(x+1)+ camIDs.append(singleCamID) #creates IDs for each camera based on the number of cameras entered+ singleVidName = 'raw_cam{}.mp4'.format(x+1)+ videoNames.append(singleVidName)++ threads = []++ for n in numCamRange: #starts recording video, opens threads for each camera+ camRecordings = CamThread(camIDs[n],camInputs[n],videoNames[n],rawVidPath,beginTime,parameterDictionary)+ camRecordings.start()++ threads.append(camRecordings)++ for camRecordings in threads:+ camRecordings.join() #make sure that one thread ending doesn't immediately end all the others (before they can dump data in a pickle file)++ print('finished')++ timeStampList = []++ for e in numCamRange: #open the saved pickle file for each camera, and add the timestamps to the dataList list+ with open(camIDs[e], 'rb') as f:+ camTimeList = pickle.load(f)+ timeStampList.append(camTimeList)++ timeDictionary = {} #our dictionary++ id_and_time = zip(camIDs,timeStampList)++ for cam,data in id_and_time:+ timeDictionary[cam] = np.array(data) #create a dictionary that holds the timestamps for each camera+++ df = pd.DataFrame.from_dict(timeDictionary, orient = 'index') #create a data frame from this dictionary+ dfT = df.transpose()+ csvPath = Path(config_settings['Paths']['sessionPath'])/csvName+ dfT.to_csv(csvPath) #turn dataframe into a CSV+++ frameTable,timeTable,frameRate,resultsTable,plots = TimeSync(dfT,numCamRange,camIDs) #start the timesync process+ #this message shows you your percentages and asks if you would like to continue or not. shuts down the program if no+ root = Tk()++ proceed = proceedGUI(root,resultsTable,plots)+ root.mainloop()+++ print()+ print('Starting editing')+ #start editing the videos+ VideoEdit(rawVidPath,syncedVidPath,videoNames,sessionName,frameTable,parameterDictionary,rotationInput,numCamRange)+++ print('all done')++def TestDevice(source):+ cap = cv2.VideoCapture(source,cv2.CAP_DSHOW)+ #if cap is None or not cap.isOpened():+ #print('Warning: unable to open video source: ', source)++ if cap.isOpened():+ #print('Opened: ',source)+ #print('Exposure: '+ str(cap.get(cv2.CAP_PROP_EXPOSURE)))+ #time.sleep(3)+ cap.release()+ cv2.destroyAllWindows()+ open_cam = source+ return open_cam+ else:+ return None++def CheckCams():+ openCamList = []+ for x in tqdm(range(20)):#range 20 right now to be safe+ openCamera = TestDevice(x)+ if openCamera is not None:+ openCamList.append(openCamera)++ return openCamList++class VideoSetup(threading.Thread):+ def __init__(self, camID,parameterDictionary,rotNum):+ self.camID = camID+ self.parameterDictionary = parameterDictionary+ self.rotNum = rotNum+ threading.Thread.__init__(self)+ def run(self):+ #print("Starting " + self.previewName)+ self.record(self.parameterDictionary,self.rotNum)+ def record(self,parameterDictionary,rotNum):+ exposure = parameterDictionary.get('exposure')+ resWidth = parameterDictionary.get('resWidth')+ resHeight = parameterDictionary.get('resHeight')+ camName = "Camera" + str(self.camID)++ cv2.namedWindow(camName)+ cap = cv2.VideoCapture(self.camID,cv2.CAP_DSHOW)+ cap.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)+ cap.set(cv2.CAP_PROP_FRAME_HEIGHT, resHeight)+ cap.set(cv2.CAP_PROP_EXPOSURE, exposure)+ #print('exposure',cap.get(cv2.CAP_PROP_EXPOSURE))+ #if not cap.isOpened():+ # raise RuntimeError('No camera found at input '+ str(self.camID))+ while True:+ ret1,frame1 = cap.read()+ if ret1 ==True:+ if rotNum is not None:+ frame1 = imutils.rotate_bound(frame1, angle= rotNum)+ cv2.imshow(camName,frame1)+ if cv2.waitKey(1) & 0xFF== 27:+ # == ord('q') for q+ break++ else:+ break+ cv2.destroyWindow(camName)+++#function to trim our videos+++++def DebugTime(sessionName,recordPath):++ def CloseNeighb(camera,point):+ closestPoint = (np.abs(camera - point)).argmin()+ return closestPoint+ csvName = sessionName + '.csv'+ csvPath = recordPath/csvName+ newFrame = pd.read_csv (csvPath)+ newFrame = newFrame.iloc[:,1:]+ camNames = list(newFrame.columns)+ numCamRange = range(len(camNames))+++++ #this section auto-finds the start and end points for our master timeline to the nearest second+ masterTimelineBegin = np.ceil(max(newFrame.iloc[0]))+ lastPoints = []+ for name in camNames:+ cameraLastPoint = newFrame[name][newFrame[name].last_valid_index()] #find the last non-nan value in each camera+ lastPoints.append(cameraLastPoint)++ masterTimelineEnd = np.floor(min(lastPoints)) #find where the fastest camera ended, round down, use that as the end point+++ #print('end',endList)+ print('intervals:',masterTimelineBegin,masterTimelineEnd)+++ totalFrameIntvl= [] #list for storing frame rates for each camera+++ n = 0; #counter for going through camera names+ for x in numCamRange:+ currentCam = newFrame[camNames[x]]+ #finds the closest point in each camera to the start and end points of our timeline+ currentCamStart = CloseNeighb(currentCam,masterTimelineBegin)+ currentCamEnd = CloseNeighb(currentCam,masterTimelineEnd)+ print("using frames:", currentCamStart,"-",currentCamEnd,"for",camNames[n])+ print(n)+ n +=1+ currentCamTimeline = currentCam[currentCamStart:currentCamEnd] #grab the times from start to finish for each camera+ currentCamFrameInterval = np.mean(np.diff(currentCamTimeline)) #calculate the interval between each frame and take the mean+ totalFrameIntvl.append(currentCamFrameInterval) #add interval to list+ #print(camNames[x],currentCamStart,currentCamEnd)++ totalAverageIntvl = np.mean(totalFrameIntvl) #find the total average interval across all cameras+ masterTimeline = np.arange(masterTimelineBegin,masterTimelineEnd,totalAverageIntvl) #build a master timeline with the average interval++ #now we start the syncing process++ frameList = masterTimeline #start a list of frames, with the first row being our master timeline+ timeList = masterTimeline #start a list of timestamps,with the first row being our master timeline++ delNum= []; #stores number of deleted frames/camera+ bufNum= []; #stores number of buffered frames/camera+ bufPercentList = []; #stores percentage of deleted frames/camera+ delPercentList = []; #stores percentage of buffered frames/camera++ count = 0 #Keeps track of what frame we're on (I think?)+ n = 0; #I don't remember why I did this but I'm sure I'll figure it out later++ for y in numCamRange:+ thisCam = newFrame[camNames[y]]+ camTimes = []; #stored times+ camFrames =[]; #stored frames++ count += 1+ #--------------Adjust each camera to start at the first point in the master timeline+ beginFrame = CloseNeighb(thisCam,masterTimeline[0])+ timeDif = masterTimeline[0] - thisCam[beginFrame]+ thisCam = thisCam + timeDif++ for z in masterTimeline: #for each point in the master timeline+ closestFrame = CloseNeighb(thisCam, z) #find the closest frame in this camera to each point of the master timeline+ camFrames.append(closestFrame) #add that frame to the list+ camTimes.append(thisCam[closestFrame]) #find the time corresponding to this frame++ print("starting detection:",camNames[n]) #now to start finding deleted/buffered slides+ frameList = np.column_stack((frameList,camFrames)) #update our framelist with our new frames+ timeList = np.column_stack((timeList,camTimes)) #update our timelist+++ #start counters for the number of buffered and deleted slides+ bufCount = 0;+ delCount = 0;++ for i in range(0,len(camFrames)-1):+ distanceBetweenFrames = camFrames[i+1] - camFrames[i] #find the distance between adjacent frames+ if distanceBetweenFrames == 1: #these frames are consecutive, do nothing+ None+ elif distanceBetweenFrames == 0: #we have a buffered slide+ bufCount += 1+ #this section looks at our two timepoints, and finds the which point on the master timeline is closest+ frame1 = abs(masterTimeline[i]-camTimes[i])+ frame2 = abs(masterTimeline[i+1]-camTimes[i])+ #finds which frame is closest, and sets the other frame as a buffer (indicated by the -1)+ if frame1>frame2:+ frameList[i,count] = -1+ timeList[i,count] = -1+ else:+ frameList[i+1,count] = -1;+ timeList[i+1,count] = -1;+ elif distanceBetweenFrames > 1: #deletion+ delCount += 1+ else:++ print("something else happened")+ #update and calculate our percentages and numbers for buffers/deletions+ delNum.append(round(delCount,1))+ bufNum.append(round(bufCount,1))+ #print(bufCount,len(frameList))+ bufPercent = (bufCount/len(frameList))*100+ delPercent = (delCount/len(frameList))*100+ bufPercentList.append(round(bufPercent,2))+ delPercentList.append(round(delPercent,2))++ #print(delNum,bufNum,bufPercentList,delPercentList)+ #print("deleted frames:",delCount)+ #print("buffered frames:",bufCount)+ n +=1++ #create our data frame for both times and frames+ frameTable = pd.DataFrame(frameList)+ columnNames = ['Master Timeline'] + camNames+ frameTable.columns = columnNames+ timeTable = pd.DataFrame(timeList)+ timeTable.columns = columnNames+ totalFrameRate = [round(1/intvl,1) for intvl in totalFrameIntvl]+ frameRate = 1/totalAverageIntvl #calculates our framerate+ results = {'Cam':camNames,'#Del':delNum,'%Del':delPercentList,'#Buf':bufNum,'%Buf':bufPercentList,'FPS':totalFrameRate}+ resultTable = pd.DataFrame(results,columns = ['Cam','#Del','%Del','#Buf','%Buf','FPS'])++ #============================================== plot data+ differenceFrame = newFrame.diff(axis = 0)+ fpsFrame = differenceFrame.pow(-1)+ fig = Figure(figsize = [10,10])+ fig.patch.set_facecolor('#F0F0F0')+ a = fig.add_subplot(221)+ a.set_xlabel('Frame')+ a.set_ylabel('Time(s)')+ newFrame.plot(ax = a, title = 'Camera timestamps')+ b = fig.add_subplot(222)+ b.set_xlabel('Frame')+ b.set_ylabel('Frame duration (s)')+ differenceFrame.plot(ax = b, title = 'Camera frame duration')+ c = fig.add_subplot(223)+ c.set_xlabel('Time (s)')+ differenceFrame.plot.hist(ax = c, bins=6,alpha = .5, xlim = [0,.1], title = 'Frame interval distribution (s)')+ d = fig.add_subplot(224)+ d.set_xlabel('Frames per second')+ fpsFrame.plot.hist(ax = d, bins=6,alpha = .5, xlim = [0,40], title = 'Frames per second distribution')+ root = Tk()++ proceedGUI(root,resultTable,fig)+ root.mainloop()\ No newline at end of filediff --git a/freemocap/webcamrecord.py b/freemocap/webcamrecord.pynew file mode 100644index 00000000..bd1c3dba--- /dev/null+++ b/freemocap/webcamrecord.py@@ -0,0 +1,108 @@+# -*- coding: utf-8 -*-+"""+Created on Mon Dec 7 14:16:45 2020++@author: Rontc+"""+++from freemocap import recordingconfig+from freemocap.webcam import recordGUI,camsetup,runrecordingprocess++import tkinter as tk+from pathlib import Path+import datetime+import os+from ruamel.yaml import YAML++def recordWebcamVideos(session):+ #%% -----------------------------------------------SESSION INFO+ #Choose a file path+ userPath = '' #add custom path here if desired as r'[filepath]\', and for right now **you should have two backslashes at the end of your path**+ if not userPath:+ filepath = Path.cwd()+ else:+ filepath = userPath++ #create session ID+ sessionID_in = datetime.datetime.now().strftime("sesh_%y-%m-%d_%H%M%S")+ session.sessionID = sessionID_in++ #load the parameters YAML and extract the saved parameters if possible, and the default otherwise+ parameter_path = filepath/'parameters.yaml'+ parameters_yaml = YAML()+ parameters = parameters_yaml.load(parameter_path)++ try:+ rotation_entry = parameters['saved']['rotations']+ parameter_entry = parameters['saved']['parameters']+ except:+ print("Using Default Config")+ rotation_entry = parameters['default']['rotations']+ parameter_entry = parameters['default']['parameters']+ #---------------------------------------------Run With GUI++ #run the GUI to get the tasks, the cams chosen, the camera settings, and the session ID+ task, cam_inputs, rotDict, paramDict, session.sessionID = recordGUI.RunGUI(sessionID_in,rotation_entry,parameter_entry)+++ #update the saved parameters in the YAML+ parameters['saved']['rotations'] = rotDict+ parameters['saved']['parameters'] = paramDict+ parameters_yaml.dump(parameters, parameter_path)++ #create a list from the rotation dictionary to be used in running webcams+ rotation_input = list(rotDict.values())++ proceedToCalibration = ''+ # %% TASK CHOICES++ #-----------------------------------------------SETUP+ #when testing, press 'escape' to individually exit each feed. Camera input number associated with feed is displayed up top++ if task == 'setup': #don't change this boolean by accident pls++ if not cam_inputs:+ raise ValueError('Camera input list (cam_inputs) is empty')++ camsetup.RunSetup(cam_inputs,rotation_input,paramDict)+ continueToRecording,session.sessionID = recordGUI.RunProceedtoRecordGUI(sessionID_in)++ #proceedToCalibration = ''++ if continueToRecording:+ config_yaml_path = recordingconfig.createSession(session,filepath)+ recordingconfig.createSessionTxt(session,paramDict,rotDict)+ proceedToCalibration = runrecordingprocess.RunCams(session,cam_inputs,paramDict,rotation_input) #press ESCAPE to end the recording++ #-----------------------------------------------RECORD+ #Press ESCAPE to stop the recording process, and continue onto the time-syncing/editing process++ elif task == 'record':#don't change this boolean by accident pls+ #if continueToRecording:+ #create a recording session+ config_yaml_path = recordingconfig.createSession(session,filepath)+ recordingconfig.createSessionTxt(session,paramDict,rotDict)++ #check that cams were chosen (these lines can be removed when we make it mandatory to use the GUI)+ if not cam_inputs:+ raise ValueError('Camera input list (cam_inputs) is empty')++ #run the recording webcams and get back a table of results+ proceedToCalibration = runrecordingprocess.RunCams(session,cam_inputs,paramDict,rotation_input) #press ESCAPE to end the recording+++ elif not task:+ print('No task detected')+++ elif task == 'debug': #out-of-order for right now+ #to debug, manually enter a sessionID and change the task, and the results table/figures from that session will be generated+ recordPath = filepath/'Data'/sessionID+ webcam.DebugTime(sessionID,recordPath)+++ return proceedToCalibration++#if __name__ == "__main__":+# runCams()\ No newline at end of filediff --git a/parameters.yaml b/parameters.yamlnew file mode 100644index 00000000..e6236323--- /dev/null+++ b/parameters.yaml@@ -0,0 +1,24 @@+#default parameters+++default:+ rotations: {}+ parameters:+ exposure: -5+ resWidth: 640+ resHeight: 480+ framerate: 25+ codec: DIVX++#saved parameters++++saved:+ rotations:+ parameters:+ exposure:+ resWidth:+ resHeight:+ framerate:+ codec:diff --git a/runme_fmc_staged.py b/runme_fmc_staged.pynew file mode 100644index 00000000..95610276--- /dev/null+++ b/runme_fmc_staged.py@@ -0,0 +1,84 @@+from freemocap import createvideo, initialization, runcams, calibrate, runopenpose, reconstruct3D, playskeleton, session+++from aniposelib.boards import CharucoBoard, Checkerboard++import numpy as np+++# %% Inputs to edit++stage = 8 #set your starting stage here (stage = 1 will run the pipeline from webcams)++board = CharucoBoard(7, 5,+ #square_length=1, # here, in mm but any unit works (JSM NOTE - just using '1' so resulting units will be in 'charuco squarelenghts`)+ #marker_length=.8,+ square_length = 7,+ marker_length = 6.8,+ marker_bits=4, dict_size=250)+++sesh = session.Session(useOpenPose=True,useDLC=False)+sesh.input_stage = stage+sesh.debug = True+sesh.sessionID = 'sesh_21-05-28_113930' #fill in if you are running from Stage 2 onwards+++# %% Initialization++initialization.initialize(sesh,stage,board)++# %% Stage One+if not stage > 1:+ runcams.RecordCams(sesh, sesh.cam_inputs, sesh.parameterDictionary, sesh.rotationInputs)+else:+ print('Skipping Video Recording')+++# %% Stage Two+if not stage > 2:+ runcams.SyncCams(sesh, sesh.timeStampData,sesh.numCamRange,sesh.vidNames,sesh.camIDs)+else:+ print('Skipping Video Syncing')++if not stage > 3:+ sesh.cgroup, sesh.mean_charuco_fr_mar_dim = calibrate.CalibrateCaptureVolume(sesh,board)+else:+ print('Skipping Calibration')++if not stage > 4:+ runopenpose.runOpenPose(sesh)+else:+ print('Skipping Running OpenPose')++if not stage > 5:+ sesh.openPoseData_nCams_nFrames_nImgPts_XY = runopenpose.parseOpenPose(sesh)+else:+ print('Skipping Parse OpenPose')++++if not stage > 6:+ openPose_params = sesh.openPoseData_nCams_nFrames_nImgPts_XY.shape[0:3]+ sesh.skel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XY, openPose_params)++ path_to_skel_points = sesh.dataArrayPath/'skeleton_points.npy'+ np.save(path_to_skel_points, sesh.skel_fr_mar_dim)+else:+ print('Skipping Skeleton Reconstruction')+++if not stage > 7:+ playskeleton.ReplaySkeleton(sesh,1,40,-90,-75)+else:+ print('Skipping Skeleton Plotting')+++if not stage > 8:+ createvideo.createVideo(sesh)+else:+ print('Skipping Video Creation')++++f = 2\ No newline at end of file