In the previous script, we have prepared all the videos as trial-sized files that we can use for motion capture. However, during the experimental recording, we have concatenated the three cameras into one file. Now we need to cut these videos into three and prepare them into folders as OpenPose (and later Pose2sim) requires.
Note that the folder structure needed for motion tracking changed with new version of Pose2sim, therefore this scripts distributes the data into folders differently than it was preregistered.
We will do the same for calibration videos.
Code to setup the environment
import osimport cv2import globimport tempfileimport subprocessimport randomfrom IPython.display import Video# Currect foldercurfolder = os.getcwd()# Videodata videodata = os.path.join(curfolder, '..', '01_XDF_processing', 'data', 'Data_processed', 'Data_trials')# Here we store all trialsoutputfolder = os.path.join(curfolder, 'projectdata')ifnot os.path.exists(outputfolder): os.makedirs(outputfolder)# Load in the videos (avi)videos = []forfilein os.listdir(videodata):iffile.endswith(".avi"):# check if it has _pr_, these are practice and we can ignore themif'_pr_'notinfile: videos.append(os.path.join(videodata, file))print(videos[0:10])# Calibration videos are in rawdata foldercalibfolder = os.path.join(curfolder, '..', '00_raw')# Extrinsic calibrationvideos_ex = glob.glob(os.path.join(calibfolder, '*', '*checker*.avi'), recursive=True)videos_ex = [video for video in videos_ex if'charuco_cut'notin video] # get rid of all that have '*charuco_cut in name# Intrinsic calibrationvideo_in = glob.glob(os.path.join(calibfolder, '*', '*intrinsics.avi'), recursive=True) # we don't need to do that anymore
This is how the video looks like when the cameras are still concatenated
Function to split the videos into 3 camera views
def split_camera_views(input_file, output_files):""" Splits a video into three separate camera views by dividing the width into three equal parts. Parameters: ----------- input_file : str Path to the input video file. output_files : list of str List of three output file paths for the three camera views. """ cap = cv2.VideoCapture(input_file)# Divide the width by 3 to get each camera separately num_cameras =3 width_per_camera =int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) // num_cameras height =int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) frame_rate =int(cap.get(cv2.CAP_PROP_FPS))# Create VideoWriters for each camera fourcc = cv2.VideoWriter_fourcc(*'XVID') out_cam1 = cv2.VideoWriter(output_files[0], fourcc, frame_rate, (width_per_camera, height)) out_cam2 = cv2.VideoWriter(output_files[1], fourcc, frame_rate, (width_per_camera, height)) out_cam3 = cv2.VideoWriter(output_files[2], fourcc, frame_rate, (width_per_camera, height))whileTrue: ret, frame = cap.read()# Check if the frame is None (end of video)if frame isNone:break# Break the frame into three parts camera1_frame = frame[:, :width_per_camera, :] camera2_frame = frame[:, width_per_camera:2*width_per_camera, :] camera3_frame = frame[:, 2*width_per_camera:, :]# Display each camera view separately (optional) cv2.imshow('Camera 1', camera1_frame) cv2.imshow('Camera 2', camera2_frame) cv2.imshow('Camera 3', camera3_frame)# Write frames to video files out_cam1.write(camera1_frame) out_cam2.write(camera2_frame) out_cam3.write(camera3_frame)if cv2.waitKey(1) ==27:break# Release VideoWriters and VideoCapture out_cam1.release() out_cam2.release() out_cam3.release() cap.release() cv2.destroyAllWindows()
Cutting trial videos
# Loop over files in folder and split themforfilein videos:print("working on file: "+file)# Get the name of the file without the extension filename = os.path.splitext(os.path.basename(file))[0]# Get trialID# If it's a tpose, the name goes a bit differently than the restif'tpose'in filename: if'rein'in filename:# If it's a rein, we take the session, part, trial number and participant as trial ID trialID = filename.split("_")[0] +"_"+ filename.split("_")[1] +"_"+ filename.split("_")[2] +"_"+ filename.split("_")[3] +"_"+ filename.split("_")[4]else: trialID = filename.split("_")[0] +"_"+ filename.split("_")[1] +"_"+ filename.split("_")[2]+"_"+ filename.split("_")[3]else:if'rein'in filename:# If it's a rein, we take the session, part, trial number and participant as trial ID trialID = filename.split("_")[0] +"_"+ filename.split("_")[1] +"_"+ filename.split("_")[2] +"_"+ filename.split("_")[4] +"_"+ filename.split("_")[5]else:# session, part, trial number and participant as trial ID trialID = filename.split("_")[0] +"_"+ filename.split("_")[1] +"_"+ filename.split("_")[3] +"_"+ filename.split("_")[4]# Get sessionID sessionID ='Session'+'_'+ filename.split("_")[0] +"_"+ filename.split("_")[1]# If a sessionID folder doesn't exist yet, create itifnot os.path.exists(os.path.join(outputfolder, sessionID)): os.makedirs(os.path.join(outputfolder, sessionID)) # Now trialID folder within respective participantif'p0'in filename or'tpose_0'in filename or'p1'in filename or'tpose_1'in filename:# create p0 folder with trial id if it doesn't existifnot os.path.exists(os.path.join(outputfolder, sessionID, trialID)): os.makedirs(os.path.join(outputfolder, sessionID, trialID))# Inside this folder, create empty folder 'raw-2d'ifnot os.path.exists(os.path.join(outputfolder, sessionID, trialID, 'raw-2d')): os.makedirs(os.path.join(outputfolder, sessionID, trialID, 'raw-2d'))# If in this folder are 3 videos already, skipiflen(os.listdir(os.path.join(outputfolder, sessionID, trialID, 'raw-2d'))) >=3:continue# This is how the final video is named output_files = [ os.path.join(outputfolder, sessionID, trialID, 'raw-2d', filename +'_cam1.avi'), os.path.join(outputfolder, sessionID, trialID, 'raw-2d', filename +'_cam2.avi'), os.path.join(outputfolder, sessionID, trialID, 'raw-2d', filename +'_cam3.avi') ]else:print(f"File {filename} does not contain p0 or p1, error...")break# Split the camera views split_camera_views(file, output_files)#break
Now we check if all trial folders have three videos available
folderstotrack = glob.glob(os.path.join(curfolder,'projectdata','*'))# Initiate empty listpcnfolders = []# Get all the folders per session, per participantfor i in folderstotrack: pcnfolders_in_session = glob.glob(os.path.join(i, '*')) pcnfolders = pcnfolders + pcnfolders_in_session# Get rid of all pontetially confusing files/folderspcnfolders = [x for x in pcnfolders if'Config'notin x]pcnfolders = [x for x in pcnfolders if'opensim'notin x]pcnfolders = [x for x in pcnfolders if'xml'notin x]pcnfolders = [x for x in pcnfolders if'ResultsInverseDynamics'notin x]pcnfolders = [x for x in pcnfolders if'ResultsInverseKinematics'notin x]pcnfolders = [x for x in pcnfolders if'sto'notin x]pcnfolders = [x for x in pcnfolders if'txt'notin x]# Go through every pcnfolder and check if there is raw-2d folder with 3 videosfor pcnfolder in pcnfolders:if os.path.exists(os.path.join(pcnfolder, 'raw-2d')): videos_2d = glob.glob(os.path.join(pcnfolder, 'raw-2d', '*.avi'))iflen(videos_2d) !=3:print(f"Folder {pcnfolder} does not have 3 videos in raw-2d folder, it has {len(videos_2d)}")
Cutting calibration videos
Now we also need to cut the video for calibration.
In the very beginning, we need to calibrate intrinsic parameters of the cameras (e.g., focal length). This needs to be done only once, and for that purpose we have special calibration video that is optimized for a low intrinsic error. For extrinsic parameters, we have calibration per each session.
Note that we do not need to calibrate the intrinsics again as the cameras are the same as in pilot data. We will just copy this file into all sessions in Pose2sim script
# Loop over files in folder and split themforfilein video_in:# Get the name of the file without the extension filename = os.path.splitext(os.path.basename(file))[0]# Note that we save it only to the first because intrinsic calibration needs to be done only once sessionID ='Session_1_1'# Inside this folder, create empty folder 'calibration'ifnot os.path.exists(os.path.join(outputfolder, sessionID, 'calibration')): os.makedirs(os.path.join(outputfolder, sessionID, 'calibration'))# Inside, make three folders: cam1, cam2, cam3 os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics', 'cam1')) os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam2')) os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam3'))# Create the output file names output_files = [ os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam1', filename +'_cam1.avi'), os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam2', filename +'_cam2.avi'), os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam3', filename +'_cam3.avi') ]# Split the camera views split_camera_views(file, output_files)
Now we can also cut the video for extrinsic calibration
# Loop over files in folder and split themforfilein videos_ex:print("working on file: "+file)# Get the name of the file without the extension filename = os.path.splitext(os.path.basename(file))[0] sessionID = filename.split("_")[0]# Note that we save it only to session x_1 because then we will just copy the finished calibratio toml file sessionID ='Session_'+ sessionID +"_1"# Inside this folder, create empty folder 'calibration' ifnot os.path.exists(os.path.join(outputfolder, sessionID, 'calibration')): os.makedirs(os.path.join(outputfolder, sessionID, 'calibration'))# Inside, make three folders: cam1, cam2, cam3ifnot os.path.exists(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics')): os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics'))ifnot os.path.exists(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics', 'cam1')): os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics', 'cam1'))ifnot os.path.exists(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam2')): os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam2'))ifnot os.path.exists(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam3')): os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam3'))# Create the output file names output_files = [ os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam1', filename +'_cam1.avi'), os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam2', filename +'_cam2.avi'), os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam3', filename +'_cam3.avi') ]# Split the camera views split_camera_views(file, output_files)
Now we are ready to proceed to motion tracking with OpenPose.