Motion tracking I: Preparation of videos

Overview

In the previous script, we have prepared all the videos as trial-sized files that we can use for motion capture. However, during the experimental recording, we have concatenated the three cameras into one file. Now we need to cut these videos into three and prepare them into folders as OpenPose (and later Pose2sim) requires.

We will do the same for calibration videos.

Code to setup the environment

import os
import cv2
import glob
import tempfile
import subprocess
import random
from IPython.display import Video

# Currect folder
curfolder = os.getcwd()

# Videodata 
videodata = curfolder + '\\..\\01_XDF_processing\\data\\Data_processed\\Data_trials'

# If it doesn't exist, create folder projectdata
if not os.path.exists(curfolder + '\\projectdata\\'):
    os.makedirs(curfolder + '\\projectdata\\')

# Refer to it  
outputfolder = curfolder + '\\projectdata\\'

# Load in the videos (avi)
videos = []
for file in os.listdir(videodata):
    if file.endswith(".avi"):
        videos.append(os.path.join(videodata, file))

print(videos[0:10])

# Calibration videos are in rawdata folder
calibfolder = curfolder + '\\..\\00_RAWDATA\\'

# Extrinsic calibration
videos_ex = glob.glob(calibfolder + '*\\*extrinsics.avi', recursive=True)

# Intrinsic calibration
video_in = glob.glob(calibfolder + '*\\*intrinsics.avi', recursive=True)

First, we need to take care that we are not working with wrongly cut videos, but only with the corrected version (if it exists). From the list of videos, we will hence exclude all videos that have also corrected version in the list.

# Get all corrected videos from the list
videos_corr = []
for file in videos:
    if 'corrected' in file:
        videos_corr.append(file)

# Now get the name of this trial without the corrected part
videos_old = []
for file in videos_corr:
    videos_old.append(file.replace('_corrected', ''))

# From videos, remove trials that are in videos_old
videos = [x for x in videos if x not in videos_old]

This is how the video looks like when the cameras are still concatenated

Function to split the videos into 3 camera views

def split_camera_views(input_file, output_files):
    cap = cv2.VideoCapture(input_file)

    # Divide the width by 3 to get each camera separately
    num_cameras = 3
    width_per_camera = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) // num_cameras
    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    frame_rate = int(cap.get(cv2.CAP_PROP_FPS))

    # Create VideoWriters for each camera
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    out_cam1 = cv2.VideoWriter(output_files[0], fourcc, frame_rate, (width_per_camera, height))
    out_cam2 = cv2.VideoWriter(output_files[1], fourcc, frame_rate, (width_per_camera, height))
    out_cam3 = cv2.VideoWriter(output_files[2], fourcc, frame_rate, (width_per_camera, height))

    while True:
        ret, frame = cap.read()

        # Check if the frame is None (end of video)
        if frame is None:
            break

        # Break the frame into three parts
        camera1_frame = frame[:, :width_per_camera, :]
        camera2_frame = frame[:, width_per_camera:2*width_per_camera, :]
        camera3_frame = frame[:, 2*width_per_camera:, :]

        # Display each camera view separately (optional)
        cv2.imshow('Camera 1', camera1_frame)
        cv2.imshow('Camera 2', camera2_frame)
        cv2.imshow('Camera 3', camera3_frame)

        # Write frames to video files
        out_cam1.write(camera1_frame)
        out_cam2.write(camera2_frame)
        out_cam3.write(camera3_frame)

        if cv2.waitKey(1) == 27:
            break

    # Release VideoWriters and VideoCapture
    out_cam1.release()
    out_cam2.release()
    out_cam3.release()
    cap.release()
    cv2.destroyAllWindows()

Cutting trial videos

We will first start with the trial videos, as they require a bit different structuring into folders than the calibration videos. Each video-tryad will be saved into following structure /sessionID/pcnID/trialID/raw-2d with its original name and identificator of each camera (i.e., 1-3)

# Loop over files in folder and split them
for file in videos:
    print("working on file: "+ file)

    # Get the name of the file without the extension
    filename = os.path.splitext(os.path.basename(file))[0]
    
    # Get trialID
    # If it's a tpose, the name goes a bit differently than the rest
    if 'tpose' in filename: 
        trialID = filename.split("_")[0] + "_" + filename.split("_")[1] + "_" + filename.split("_")[2]+ "_p" + filename.split("_")[3]
    else:
        # session, part, trial number and participant as trial ID
        trialID = filename.split("_")[0] + "_" + filename.split("_")[1] + "_" + filename.split("_")[3] + "_" + filename.split("_")[4]

    # Get sessionID
    sessionID = 'Session' + '_' + filename.split("_")[0] + "_" + filename.split("_")[1]

    # If a sessionID folder doesn't exist yet, create it
    if not os.path.exists(os.path.join(outputfolder, sessionID)):
        os.makedirs(os.path.join(outputfolder, sessionID))    

    # Same for folders P0 and P1
    if not os.path.exists(os.path.join(outputfolder, sessionID, 'P0')):
        os.makedirs(os.path.join(outputfolder, sessionID, 'P0'))
    if not os.path.exists(os.path.join(outputfolder, sessionID, 'P1')):
        os.makedirs(os.path.join(outputfolder, sessionID, 'P1'))

    # Now trialID folder within respective participant
    if 'p0' in filename or 'tpose_0' in filename:
        try:
            os.makedirs(os.path.join(outputfolder, sessionID, 'P0', trialID))
            # Inside this folder, create empty folder 'raw-2d'
            os.makedirs(os.path.join(outputfolder, sessionID, 'P0', trialID, 'raw-2d'))
        except FileExistsError:
            continue

        # This is how the final video is named
        output_files = [
            os.path.join(outputfolder, sessionID, 'P0', trialID, 'raw-2d', filename + '_cam1.avi'),
            os.path.join(outputfolder, sessionID, 'P0', trialID, 'raw-2d', filename + '_cam2.avi'),
            os.path.join(outputfolder, sessionID, 'P0', trialID, 'raw-2d', filename + '_cam3.avi')
        ]

    elif 'p1' in filename or 'tpose_1' in filename:
        try:
            os.makedirs(os.path.join(outputfolder, sessionID, 'P1', trialID))
            os.makedirs(os.path.join(outputfolder, sessionID, 'P1', trialID, 'raw-2d'))
        except FileExistsError:
            continue    

        # This is how the final video is named
        output_files = [
            os.path.join(outputfolder, sessionID, 'P1', trialID, 'raw-2d', filename + '_cam1.avi'),
            os.path.join(outputfolder, sessionID, 'P1', trialID, 'raw-2d', filename + '_cam2.avi'),
            os.path.join(outputfolder, sessionID, 'P1', trialID, 'raw-2d', filename + '_cam3.avi')
        ]

    else:
        try:
            os.makedirs(os.path.join(outputfolder, sessionID, trialID))
            os.makedirs(os.path.join(outputfolder, sessionID, trialID, 'raw-2d'))
        except FileExistsError:
            continue

        output_files = [
            os.path.join(outputfolder, sessionID, trialID, 'raw-2d', filename + '_cam1.avi'),
            os.path.join(outputfolder, sessionID, trialID, 'raw-2d', filename + '_cam2.avi'),
            os.path.join(outputfolder, sessionID, trialID, 'raw-2d', filename + '_cam3.avi')]

    # Split the camera views
    split_camera_views(file, output_files)

Cutting calibration videos

Now we also need to cut the video for calibration.

In the very beginning, we need to calibrate intrinsic parameters of the cameras (e.g., focal length). This needs to be done only once, and for that purpose we have special calibration video that is optimized for a low intrinsic error. For extrinsic parameters, we have calibration per each session.

# Loop over files in folder and split them
for file in video_in:
    # Get the name of the file without the extension
    filename = os.path.splitext(os.path.basename(file))[0]
    
    # sessionID
    sessionID = filename.split("_")[0]
    # note that we save it only to session 0_1 because intrinsic calibration needs to be done only once
    sessionID = 'Session_' + sessionID + "_1" 

    # Inside this folder, create empty folder 'calibration'
    if not os.path.exists(os.path.join(outputfolder, sessionID, 'calibration')):
        os.makedirs(os.path.join(outputfolder, sessionID, 'calibration'))
    # Inside, make three folders: cam1, cam2, cam3
    os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics', 'cam1'))
    os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam2'))
    os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam3'))
    
    # Create the output file names
    output_files = [
        os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam1', filename + '_cam1.avi'),
        os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam2', filename + '_cam2.avi'),
        os.path.join(outputfolder, sessionID, 'calibration', 'intrinsics','cam3', filename + '_cam3.avi')
    ]
    
    # Split the camera views
    split_camera_views(file, output_files)

Now we can also cut the video for extrinsic calibration

# Loop over files in folder and split them
for file in videos_ex:
    # Get the name of the file without the extension
    filename = os.path.splitext(os.path.basename(file))[0]

    sessionID = filename.split("_")[0]
    # Note that we save it only to session x_1 because then we will just copy the finished calibratio toml file
    sessionID = 'Session_' + sessionID + "_1" 

    # Inside this folder, create empty folder 'calibration' 
    if not os.path.exists(os.path.join(outputfolder, sessionID, 'calibration')):
        os.makedirs(os.path.join(outputfolder, sessionID, 'calibration'))

    # Inside, make three folders: cam1, cam2, cam3
    os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics', 'cam1'))
    os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam2'))
    os.makedirs(os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam3'))
    
    # Create the output file names
    output_files = [
        os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam1', filename + '_cam1.avi'),
        os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam2', filename + '_cam2.avi'),
        os.path.join(outputfolder, sessionID, 'calibration', 'extrinsics','cam3', filename + '_cam3.avi')
    ]
    
    # Split the camera views
    split_camera_views(file, output_files)

Now we are ready to proceed to motion tracking with OpenPose.