Pre-Processing IV: Final checks

Overview

In this script, we:

renumber reinitiated sessions such that they continue from the previous session
delete non-final cuts
check if all streams are present in each trial
create final audio-video

Code to prepare environment

import os
import glob
import pandas as pd
from multiprocessing import process
import re
import csv
from pathlib import Path
import ffmpeg

# Prepare folders
curfolder = os.getcwd()
trialfolder = os.path.join(curfolder, 'data', 'Data_processed', 'Data_trials')
cutfiles = glob.glob(os.path.join(trialfolder, '*_cut*.csv'))

print(len(cutfiles), "files to process")

# From cutfiles, extract first 4 elements, if it's rein in name then 5
trials_id = []

for f in cutfiles:
    base = os.path.basename(f)
    parts = base.split('_')
    if 'rein' in parts:
        trial_id = '_'.join(parts[:5])
    else:
        trial_id = '_'.join(parts[:4])
    trials_id.append(trial_id)

trials_id = [t + '_' for t in trials_id]
trials_id = list(set(trials_id))
print(len(trials_id), "unique trials")

7035 files to process
2345 unique trials

Code with functions

def process_interrim_cuts(trial, related_files, cuts, todel_folder):
    """
    Processes interim cut files for a given trial, keeping only the highest numbered cut and moving others to a deletion folder.

    Args:
        trial (str): The trial identifier being processed
        related_files (list): List of file paths related to this trial
        cuts (set): Set of cut numbers found for this trial
        todel_folder (str): Path to folder where files to be deleted will be moved
    """

    # Identify the highest cut number
    highest_cut = max(int(cut) for cut in cuts)
    tokeep = f"cut{highest_cut}"

    # Keep only the file with the highest cut number
    files_to_delete = [f for f in related_files if tokeep not in os.path.basename(f)]
    print(f"Keeping {tokeep}, deleting {len(files_to_delete)} files for trial {trial}")

    # Move them into folder todel
    for file in files_to_delete:
        newloc = os.path.join(todel_folder, os.path.basename(file))
        os.rename(file, newloc)
        print(f"Moved {file} to {newloc}")

Get rid of interim cuts

Each new cut is adding a number to cut*.csv. We want to leave only the final cut, so the one with highest number for that specific file.

# Create temporary folder where we move all to-be-deleted files
todel_folder = os.path.join(trialfolder, 'todel')
os.makedirs(todel_folder, exist_ok=True)

############ Process csv files ############
for trial in trials_id:

    print(f"Processing trial {trial}")
    related_files = [f for f in cutfiles if os.path.basename(f).startswith(trial)]

    # How many unique cuts there are
    cuts = set()
    for f in related_files:
        base = os.path.basename(f)
        cut_num = base.split('_cut')[1].split('.csv')[0]
        cuts.add(cut_num)

    print(f"Trial {trial} has {len(related_files)} files with cuts: {cuts}")

    # Check that the cuts value is one and it equals to 1
    if len(cuts) == 1 and '1' in cuts:
        print(f"Only one cut for trial {trial}, skipping.")
        continue

    else: 
        process_interrim_cuts(trial, related_files, cuts, todel_folder)


############### Process avi files ###############
cutavifiles = glob.glob(os.path.join(trialfolder, '*_cut*.avi'))
print(len(cutavifiles), "avi files to process")

for trial in trials_id:
    
    related_files = [f for f in cutavifiles if os.path.basename(f).startswith(trial)]
    if len(related_files) == 0:
        continue

    cuts = set()
    for f in related_files:
        base = os.path.basename(f)
        cut_num = base.split('_cut')[1].split('_video_raw.avi')[0]
        cuts.add(cut_num)

    print(f"Trial {trial} has {len(related_files)} avi files with cuts: {cuts}")

    if len(cuts) == 1 and '1' in cuts:
        print(f"Only one cut for trial {trial}, skipping.")
        continue

    else:
        process_interrim_cuts(trial, related_files, cuts, todel_folder)

########## Process denoised files ###############
cutwavfiles = glob.glob(os.path.join(trialfolder, '*', '*_cut*.wav'), recursive=True)
cutwavfiles = [f for f in cutwavfiles if 'denoised' in os.path.basename(f)]
print(len(cutwavfiles), "wav files to process")

for trial in trials_id:
    
    related_files = [f for f in cutwavfiles if os.path.basename(f).startswith(trial)]
    if len(related_files) == 0:
        continue

    cuts = set()
    for f in related_files:
        base = os.path.basename(f)
        cut_num = base.split('_cut')[1].split('_denoised.wav')[0]
        cuts.add(cut_num)

    print(f"Trial {trial} has {len(related_files)} wav files with cuts: {cuts}")

    if len(cuts) == 1 and '1' in cuts:
        print(f"Only one cut for trial {trial}, skipping.")
        continue

    else:
        process_interrim_cuts(trial, related_files, cuts, todel_folder)

############ Process original wav files ############
cutwavfiles = glob.glob(os.path.join(trialfolder, '*', '*_cut*.wav'), recursive=True)
cutwavfiles = [f for f in cutwavfiles if 'denoised' not in os.path.basename(f)]
print(len(cutwavfiles), "wav files to process")

for trial in trials_id:
    
    related_files = [f for f in cutwavfiles if os.path.basename(f).startswith(trial)]
    if len(related_files) == 0:
        continue

    cuts = set()
    for f in related_files:
        base = os.path.basename(f)
        cut_num = base.split('_cut')[1].split('.wav')[0]
        cuts.add(cut_num)

    print(f"Trial {trial} has {len(related_files)} wav files with cuts: {cuts}")

    if len(cuts) == 1 and '1' in cuts:
        print(f"Only one cut for trial {trial}, skipping.")
        continue

    else:
        process_interrim_cuts(trial, related_files, cuts, todel_folder)

Renumber reinitiated sessions

If the session was reinitiated due to recording error, the XDF processing script processed them as-if they were new sessions, starting the trial count from 0. We now want to make them such that continue from the last trial number of the preceding part of the same session to simplify later processing and analysis.

Prepare variables for re-numbering

reins = ['1_2', '8_1', '18_2', '31_2', '49_1', '49_2', '50_1', '54_2']

files = glob.glob(os.path.join(trialfolder, '*.csv'))
files += glob.glob(os.path.join(trialfolder, '*.avi'))
files += glob.glob(os.path.join(trialfolder, '*', '*.wav'), recursive=True)
files = [f for f in files if 'tpose' not in f]

files_to_renumber = []

for file in files:
    # Get the sessionid (first 2 elements)
    base = os.path.basename(file)
    parts = base.split('_')
    session_id = '_'.join(parts[:2])

    if session_id in reins:
        files_to_renumber.append(file)

print(len(files_to_renumber), "files to renumber") 

# For each file, collect the trial_ids
trial_ids = set()
for f in files_to_renumber:
    base = os.path.basename(f)
    parts = base.split('_')
    if 'rein' in parts:
        trial_id = '_'.join(parts[:5])
    else:
        trial_id = '_'.join(parts[:4])
    trial_ids.add(trial_id)

trial_ids = list(trial_ids)
trial_ids = [t + '_' for t in trial_ids]

print(len(trial_ids), "unique trial ids to renumber")

365 files to renumber
73 unique trial ids to renumber

for session in reins:

    # Collect all files of this session
    session_files = [f for f in files_to_renumber if os.path.basename(f).startswith(session+'_')]
    reinfiles = [f for f in session_files if 'rein' in os.path.basename(f)]
    nonreinfiles = [f for f in session_files if 'rein' not in os.path.basename(f)]

    # Extract from nonrein files the trial numbers (3rd element)
    trial_numbers = []

    # Order nonreinfiles by 3rd element
    nonreinfiles = sorted(nonreinfiles, key=lambda x: int(os.path.basename(x).split('_')[3]))

    for f in nonreinfiles:
        base = os.path.basename(f)
        parts = base.split('_')
        trial_num = parts[3]
        trial_numbers.append(int(trial_num))

    trial_numbers = list(set(trial_numbers))
    print(f"Session {session} has {len(trial_numbers)} non-rein trials: {trial_numbers}")

    if session != '8_1' and session != '49_2':
        # Check if all numbers go 0 to n
        expected_numbers = list(range(0, len(trial_numbers)))
        if trial_numbers != expected_numbers:
            print(f"Warning: non-rein trial numbers for session {session} are not consecutive starting from 0")
            print(f"Expected: {expected_numbers}, Found: {trial_numbers}")
            break

        last_trialnumber = trial_numbers[-1]
        print(f"Last non-rein trial number for session {session} is {last_trialnumber}")

    else:
        # For sessions 8_1 and 49_2, the non-rein trials are missing
        if session == '8_1':
            last_trialnumber = 17
        elif session == '49_2':
            last_trialnumber = 23
        else:
            print("This should not happen")
            break

    rein_trial_numbers = []

    # Order reinfiles by 4th element
    reinfiles = sorted(reinfiles, key=lambda x: int(os.path.basename(x).split('_')[4]))

    for f in reinfiles:
        base = os.path.basename(f)
        parts = base.split('_')
        trial_num = parts[4]
        rein_trial_numbers.append(int(trial_num))

    rein_trial_numbers = list(set(rein_trial_numbers))
    print(f"Session {session} has {len(rein_trial_numbers)} rein trials: {rein_trial_numbers}")

    # Check if all numbers go from 0 and +1
    expected_rein_numbers = list(range(0, len(rein_trial_numbers)))

    if rein_trial_numbers != expected_rein_numbers:
        print(f"Warning: rein trial numbers for session {session} are not consecutive starting from 0")

        if session == '18_2':
            print(f"Session {session} needs to be renumbered, proceeding...")
            
            print('Getting rid of empty gesture condition in between rein')
            for f in reinfiles:
                base = os.path.basename(f)
                parts = base.split('_')
                trial_num = int(parts[4])

                if trial_num >= 59:
                    new_trial_num = trial_num - 18
                else:
                    new_trial_num = trial_num

                parts[4] = str(new_trial_num)
                new_base = '_'.join(parts)
                new_path = os.path.join(os.path.dirname(f), new_base)
                os.rename(f, new_path)
                print(f"Renamed {f} to {new_path}")

            # Collect all files again
            files = glob.glob(os.path.join(trialfolder, '*.csv'))
            files += glob.glob(os.path.join(trialfolder, '*.avi'))
            files += glob.glob(os.path.join(trialfolder, '*', '*.wav'), recursive=True)
            
            # Get rid of tpose
            files = [f for f in files if 'tpose' not in f]

            files_to_renumber = []

            for file in files:
                # Get the sessionid (first 2 elements)
                base = os.path.basename(file)
                parts = base.split('_')
                session_id = '_'.join(parts[:2])

                if session_id in reins:
                    files_to_renumber.append(file)

            print(len(files_to_renumber), "files to renumber") 

            # Collect all files of this session
            session_files = [f for f in files_to_renumber if os.path.basename(f).startswith(session+'_')]

            # Collect reinfiles again
            reinfiles = []
            reinfiles = [f for f in session_files if 'rein' in os.path.basename(f)]

            # Order reinfiles by 4th element
            reinfiles = sorted(reinfiles, key=lambda x: int(os.path.basename(x).split('_')[4]))

            rein_trial_numbers = []
            for f in reinfiles:
                base = os.path.basename(f)
                parts = base.split('_')
                trial_num = parts[4]
                rein_trial_numbers.append(int(trial_num))

            rein_trial_numbers = list(set(rein_trial_numbers))
            print(f"Session {session} has {len(rein_trial_numbers)} rein trials: {rein_trial_numbers}")

            # Check if all numbers go from 0 and +1
            expected_rein_numbers = list(range(0, len(rein_trial_numbers)))

            if rein_trial_numbers != expected_rein_numbers:
                print(f"Warning: rein trial numbers for session {session} are not consecutive starting from 0")

        else:
            print(f"Expected: {expected_rein_numbers}, Found: {rein_trial_numbers}")
            break

    else:
        print(f"Session {session} rein trial numbers are consecutive, no renumbering needed.")

    print('Renaming rein trials now...')
    
    # Rename rein trials such that you add last_trialnumber to them
    for f in reinfiles:
        base = os.path.basename(f)
        parts = base.split('_')
        trial_num = int(parts[4])
        new_trial_num = trial_num + last_trialnumber + 1
        parts[4] = str(new_trial_num)
        new_base = '_'.join(parts)
        new_path = os.path.join(os.path.dirname(f), new_base)
        os.rename(f, new_path)
        print(f"Renamed {f} to {new_path}")

Now we are left with one cut and the original file.

Deleting uncut files

Now we can delete all the original files with incorrect cut

# Collect all cut files
cutcsvfiles = glob.glob(os.path.join(trialfolder, '*_cut*.csv'))
cutwavfiles = glob.glob(os.path.join(trialfolder, '*', '*_cut*.wav'), recursive=True)
cutwavfiles += glob.glob(os.path.join(trialfolder, '*_cut*.wav'), recursive=True)
cutdenoisedfiles = glob.glob(os.path.join(trialfolder, '*', '*_cut*denoised*.wav'), recursive=True)
cutdenoisedfiles += glob.glob(os.path.join(trialfolder, '*_cut*denoised*.csv'), recursive=True)
cutavifiles = glob.glob(os.path.join(trialfolder, '*_cut*.avi'))

# Summarize
cutfilesall = cutcsvfiles + cutwavfiles + cutdenoisedfiles + cutavifiles
cutfilesall = list(set(cutfilesall))

# Collect original files
origfilesall = glob.glob(os.path.join(trialfolder, '*.csv'))
origfilesall += glob.glob(os.path.join(trialfolder, '*', '*.wav'), recursive=True)
origfilesall += glob.glob(os.path.join(trialfolder, '*.wav'))
origfilesall += glob.glob(os.path.join(trialfolder, '*', '*denoised*.wav'), recursive=True)
origfilesall += glob.glob(os.path.join(trialfolder, '*denoised*.wav'))
origfilesall += glob.glob(os.path.join(trialfolder, '*.avi'))
origfilesall = [f for f in origfilesall if 'cut' not in os.path.basename(f)]

# Summarize
origfilesall = list(set(origfilesall))

# Create orig folder if it doesn't exist
orig_folder = os.path.join(trialfolder, 'orig_todel')
os.makedirs(orig_folder, exist_ok=True)

# Collect trial_ids
trials_id = []
for f in cutfilesall:
    base = os.path.basename(f)
    parts = base.split('_')
    if 'rein' in parts:
        trial_id = '_'.join(parts[:5])
    else:
        trial_id = '_'.join(parts[:4])
    trials_id.append(trial_id)

trials_id = [t + '_' for t in trials_id]
trials_id = list(set(trials_id))
print(len(trials_id), "unique trials")

missingcuts = []
missingorigs = []

for trial in trials_id:

    print(f"Processing trial {trial}")

    sessionid = trial.split('_')[0] + '_' + trial.split('_')[1] + '_'

    move = True

    # Find all cut files
    related_files = [f for f in cutfilesall if os.path.basename(f).startswith(trial)]
    if sessionid != '16_2_':
        if len(related_files) != 7:
            print(f"Warning: Trial {trial} does not have exactly 7 cut files, found {len(related_files)}")
            move = False
            missingcuts.append((trial, related_files))
            continue
    else:
        if len(related_files) != 6:
            print(f"Warning: Trial {trial} does not have exactly 6 cut files, found {len(related_files)}")
            move = False
            missingcuts.append((trial, related_files))
            continue

    # Find now the original files per each file, which is the same name just without _cutX
    for file in related_files:
        base = os.path.basename(file)
        if 'Mic' in base and base.endswith('.csv') :
            mic_file = file
            mic_origfile = [f for f in origfilesall if os.path.basename(f).startswith(trial) and 'Mic' in os.path.basename(f) and f.endswith('.csv')][0]
            print(mic_origfile)
        elif 'WebcamFrameStream' in base:
            webcam_file = file
            
            webcam_origfile = [f for f in origfilesall if os.path.basename(f).startswith(trial) and 'WebcamFrameStream' in os.path.basename(f)][0]
            print(webcam_origfile)
        elif 'BalanceBoard' in base:
            bb_file = file
            bb_origfile = [f for f in origfilesall if os.path.basename(f).startswith(trial) and 'BalanceBoard' in os.path.basename(f)][0]
            print(bb_origfile)

        elif 'video_raw.avi' in base:
            video_file = file
            video_origfile = [f for f in origfilesall if os.path.basename(f).startswith(trial) and 'video_raw.avi' in os.path.basename(f)][0]
            print(video_origfile)

        elif 'denoised' in base and base.endswith('.wav') and 'srate16000' in base:
            denoised_file_16 = file
            denoised_origfile = [f for f in origfilesall if os.path.basename(f).startswith(trial) and 'denoised' in os.path.basename(f) and 'srate16000' in os.path.basename(f)][0]
            print(denoised_origfile)

        elif base.endswith('.wav') and 'denoised' not in base and 'srate16000' in base:
            raw_file_16 = file
            raw_origfile_16 = [f for f in origfilesall if os.path.basename(f).startswith(trial) and 'srate16000' in os.path.basename(f) and f.endswith('.wav') and 'denoised' not in os.path.basename(f)][0]
            print(raw_origfile_16)

        elif 'srate48000' in base and base.endswith('.wav'):
            raw_file_48 = file
            raw_origfile_48 = [f for f in origfilesall if os.path.basename(f).startswith(trial) and 'srate48000' in os.path.basename(f) and f.endswith('.wav')][0]
            print(raw_origfile_48)

    # Check if all cutfiles are in
    if any(v is None for v in [mic_file, webcam_file, bb_file, video_file, denoised_file_16, raw_file_16, raw_file_48]):
        if sessionid != '16_2_':
            print(f"Warning: Trial {trial} is missing one or more cut files")
            move = False
            missingcuts.append(trial)
            continue
    
    # Check if any of the variables is none
    if any(v is None for v in [mic_origfile, webcam_origfile, bb_origfile, video_origfile, denoised_origfile, raw_origfile_16, raw_origfile_48]):
        if sessionid != '16_2_':
            print(f"Warning: Trial {trial} is missing one or more original files")
            move = False
            missingorigs.append(trial)
            continue

    if move:
        # Move original files to orig folder
        if sessionid != '16_2_':
            for orig in [mic_origfile, webcam_origfile, bb_origfile, video_origfile, denoised_origfile, raw_origfile_16, raw_origfile_48]:
                newloc = os.path.join(orig_folder, os.path.basename(orig))
                os.rename(orig, newloc)
                print(f"Moved original file {orig} to {newloc}")
        else:
            for orig in [mic_origfile, webcam_origfile, bb_origfile, video_origfile, denoised_origfile, raw_origfile_16]:
                newloc = os.path.join(orig_folder, os.path.basename(orig))
                os.rename(orig, newloc)
                print(f"Moved original file {orig} to {newloc}")

print(len(missingcuts))
print(missingcuts)


##### Aftermath ####

# Get the trial ids from missing cuts
missingcuts_ids = [t[0] for t in missingcuts]
missingcuts_sessions = [t.split('_')[0] + '_' + t.split('_')[1] for t in missingcuts_ids]
missingcuts_sessions = list(set(missingcuts_sessions))
print(len(missingcuts_sessions), "unique sessions with missing cuts")
print(missingcuts_sessions)

Check if all streams are present

curfolder = os.getcwd()
trialfolder = os.path.join(curfolder, 'Data', 'Data_processed', 'Data_trials')
wavfolder = os.path.join(trialfolder, 'Audio')
wavfolder48 = os.path.join(trialfolder, 'Audio_48')

# get all files in the trial folder
allfiles = glob.glob(os.path.join(trialfolder, '*.csv'))
allfiles = [f for f in allfiles if 'tpose' not in os.path.basename(f)]

# get all avi files
avifiles = glob.glob(os.path.join(trialfolder, '*.avi'))
avifiles = [f for f in avifiles if 'tpose' not in os.path.basename(f)]

# get all wav files
wavfiles = glob.glob(os.path.join(wavfolder, '*_denoised.wav'))
wavfiles48 = glob.glob(os.path.join(wavfolder48, '*.wav'))

# if any of these lists are empty, print a message
if len(allfiles) == 0:
    print('No files in the trial folder')
if len(avifiles) == 0:
    print('No avi files in the trial folder')
if len(wavfiles) == 0:
    print('No wav files in the trial folder')
if len(wavfiles48) == 0:
    print('No 48k wav files in the trial folder')

sessionids = []

for file in allfiles:
    # get the sessionid (first 2 elements)
    base = os.path.basename(file)
    parts = base.split('_')
    session_id = '_'.join(parts[:2])

    sessionids.append(session_id)

# keep unique
sessionids = list(set(sessionids))
print(len(sessionids), "unique sessions")

# add _ in the end of each
sessionids = [s + '_' for s in sessionids]

130 unique sessions

First we want to check if all sessions have consecutive numbering of trials

missingtrials = []

for session in sessionids:

    # Collect all files of this session
    csvfiles = [f for f in allfiles if os.path.basename(f).startswith(session)]
    wavfiles_sess = [f for f in wavfiles if os.path.basename(f).startswith(session)]
    wavfiles48_sess = [f for f in wavfiles48 if os.path.basename(f).startswith(session)]
    avifiles_sess = [f for f in avifiles if os.path.basename(f).startswith(session)]

    files = [csvfiles, wavfiles_sess, wavfiles48_sess, avifiles_sess]

    for fileset in files:
        if fileset == csvfiles:
            print('processing csv files')
        elif fileset == wavfiles_sess:
            print('processing 16k wav files')
        elif fileset == wavfiles48_sess:
            print('processing 48k wav files')
        elif fileset == avifiles_sess:
            print('processing avi files')

        if len(fileset) == 0:
            print(f"Warning: No files found for session {session} in one of the modalities")
            continue

        # Extract from nonrein files the trial numbers (3rd element)
        trial_numbers = []

        # Extract third or fourth element depending on if rein is in name
        nonreinfiles = [f for f in fileset if 'rein' not in os.path.basename(f)]

        # If there is rein in any of the files, also collect those
        if any('rein' in os.path.basename(f) for f in fileset):
            reinfiles = [f for f in fileset if 'rein' in os.path.basename(f)]
            reinfiles = sorted(reinfiles, key=lambda x: int(os.path.basename(x).split('_')[4]))

        # Order nonreinfiles by 3rd element
        nonreinfiles = sorted(nonreinfiles, key=lambda x: int(os.path.basename(x).split('_')[3]))

        for f in nonreinfiles:
            base = os.path.basename(f)
            parts = base.split('_')
            trial_num = parts[3]
            trial_numbers.append(int(trial_num))

        # If there is rein in any of the files, also collect those
        if any('rein' in os.path.basename(f) for f in fileset):
            for f in reinfiles:
                base = os.path.basename(f)
                parts = base.split('_')
                trial_num = parts[4]
                trial_numbers.append(int(trial_num))

        trial_numbers = list(set(trial_numbers))
        trial_numbers = sorted(trial_numbers)

        # Expected number of trials
        expected_trials = list(range(0, len(trial_numbers)))

        if trial_numbers != expected_trials:
            print(f"Warning: Session {session} is missing some trials, expected {expected_trials} but found {len(trial_numbers)}")
            missingtrials.append(session)
            break

        else: 
            print(f"Session {session} has all trials")

Now we check if csv files per each session has all three - WebcamFrameStream, BalanceBoard, Mic

for session in sessionids:

    # Collect all files of this session
    csvfiles = [f for f in allfiles if os.path.basename(f).startswith(session)]
    trialids = []
    
    for f in csvfiles:
        base = os.path.basename(f)
        parts = base.split('_')
        if 'rein' in parts:
            trial_id = '_'.join(parts[:5])
        else:
            trial_id = '_'.join(parts[:4])

        trialids.append(trial_id)

    trialids = list(set(trialids))
    trialids = [t + '_' for t in trialids]

    missing_webcam = []
    missing_mic = []
    missing_bb = []
    wrong_cut = []

    for trial in trialids:
        related_files = [f for f in csvfiles if os.path.basename(f).startswith(trial)]

        if len(related_files) == 0:
            print(f"Warning: No files found for trial {trial} in session {session}")
            continue

        # Check if there is cut in the first file
        if 'cut' in os.path.basename(related_files[0]):
            cut_num = int(os.path.basename(related_files[0]).split('_')[-1].replace('cut', '').replace('.csv', ''))
            # Check if all other files have the same cut number
            for f in related_files:
                if f == related_files[0]:
                    continue
                if f.endswith('.csv'):
                    if 'cut' in os.path.basename(f):
                        other_cut_num = int(os.path.basename(f).split('_')[-1].replace('cut', '').replace('.csv', ''))
                        if other_cut_num != cut_num:
                            print(f"Warning: Trial {trial} in session {session} has inconsistent cut numbers: {cut_num} and {other_cut_num}")
                            wrong_cut.append(trial)
                            continue
                        
        # Check if there is WebcamFrameStream file
        if not any('WebcamFrameStream' in os.path.basename(f) for f in related_files):
            print(f"Warning: No WebcamFrameStream file found for trial {trial} in session {session}")
            missing_webcam.append(trial)
            continue

        # Check if there is Mic file
        if not any('Mic' in os.path.basename(f) for f in related_files):
            print(f"Warning: No Mic file found for trial {trial} in session {session}")
            missing_mic.append(trial)
            continue

        # Check if there is BalanceBoard file
        if not any('BalanceBoard' in os.path.basename(f) for f in related_files):
            print(f"Warning: No BalanceBoard file found for trial {trial} in session {session}")
            missing_bb.append(trial)
            continue

        print(f"Trial {trial} in session {session} has all files")

print(len(missing_webcam), "trials with missing webcam")
print(len(missing_mic), "trials with missing mic")
print(len(missing_bb), "trials with missing bb")
print(len(wrong_cut), "trials with wrong cut")

0 trials with missing webcam
0 trials with missing mic
0 trials with missing bb
0 trials with wrong cut

Lastly, we also want to check that each WebFrameStream.csv has corresponding .avi files to prevent that there are remaining files from cutting.

# Root folder to scan (current working dir by default)
root = Path(trialfolder)

# Video examples:
#   1_1_pr_9_p1_haasten_gebaren_video_raw.avi
#   1_2_pr_27_p1_fiets_geluiden_c1_video_raw.avi
#   1_2_trial_27_p1_fiets_geluiden_c1_cut18_video_raw.avi

video_re = re.compile(
    r"""^(?P<prefix>.*?_(?:pr|trial)_\d+)_      # e.g. 1_1_pr_9 or 1_2_trial_27
         (?P<suffix>
             p\d+_[^_]+(?:_[^_]+)*              # p1_haasten_gebaren or p1_fiets_geluiden_...
             (?:_c[0-2])?                       # optional _c0/_c1/_c2
             (?:_cut\d+)?                       # optional _cutN
         )
         _video_raw(?:\.avi)?$                  # video_raw[.avi]
     """,
    re.IGNORECASE | re.VERBOSE,
)

# CSV examples:
#   1_1_pr_9_MyWebcamFrameStream_nominal_srate500_p1_haasten_gebaren.csv
#   1_2_pr_27_MyWebcamFrameStream_nominal_srate500_p1_fiets_geluiden_c1.csv
#   1_2_trial_27_MyWebcamFrameStream_nominal_srate500_p1_fiets_geluiden_c1_cut18.csv

csv_re = re.compile(
    r"""^(?P<prefix>.*?_(?:pr|trial)_\d+)_MyWebcamFrameStream_nominal_srate\d+_
         (?P<suffix>
             p\d+_[^_]+(?:_[^_]+)*              # p1_haasten_gebaren...
             (?:_c[0-2])?                       # optional _c0/_c1/_c2
             (?:_cut\d+)?                       # optional _cutN
         )
         \.csv$
     """,
    re.IGNORECASE | re.VERBOSE,
)

# Gather files
video_records = []
csv_records = []

for p in root.rglob("*"):
    if not p.is_file():
        continue
    name = p.name

    # Videos
    m_v = video_re.match(name)
    if m_v:
        key = f"{m_v.group('prefix')}_{m_v.group('suffix')}".lower()
        video_records.append({
            "key": key,
            "video_path": str(p),
            "video_name": name,
        })

    # CSVs
    if p.suffix.lower() == ".csv":
        m_c = csv_re.match(name)
        if m_c:
            key = f"{m_c.group('prefix')}_{m_c.group('suffix')}".lower()
            csv_records.append({
                "key": key,
                "csv_path": str(p),
                "csv_name": name,
            })

df_v = pd.DataFrame(video_records)
df_c = pd.DataFrame(csv_records)

if df_v.empty:
    print("No videos matching the expected pattern were found.")
    display(pd.DataFrame(columns=["video_name","video_path","csv_found","csv_path(s)"]))

else:
    # Map: key -> list of CSV paths (in case multiple CSVs per key)
    csv_map = {
        key: grp["csv_path"].tolist()
        for key, grp in df_c.groupby("key")
    }

    rows = []
    for _, r in df_v.iterrows():
        key = r["key"]
        csv_paths = csv_map.get(key, [])
        rows.append({
            "video_name": r["video_name"],
            "video_path": r["video_path"],
            "matching_key": key,
            "csv_found": len(csv_paths) > 0,
            "csv_path(s)": " | ".join(csv_paths),
        })

    df = pd.DataFrame(rows).sort_values(["csv_found", "video_name"]).reset_index(drop=True)
    display(df)

    # Only those without matching CSV
    print("\nVideos with MISSING corresponding webcam CSV:")
    missing = df[~df["csv_found"]].reset_index(drop=True)
    display(missing)

	video_name	video_path	matching_key	csv_found	csv_path(s)
0	10_1_pr_0_p0_koken_combinatie_video_raw.avi	f:\flesh_data_processed\01_XDF_processing\Data...	10_1_pr_0_p0_koken_combinatie	True	f:\flesh_data_processed\01_XDF_processing\Data...
1	10_1_pr_10_p1_gek_combinatie_video_raw.avi	f:\flesh_data_processed\01_XDF_processing\Data...	10_1_pr_10_p1_gek_combinatie	True	f:\flesh_data_processed\01_XDF_processing\Data...
2	10_1_pr_18_p0_snijden_gebaren_video_raw.avi	f:\flesh_data_processed\01_XDF_processing\Data...	10_1_pr_18_p0_snijden_gebaren	True	f:\flesh_data_processed\01_XDF_processing\Data...
3	10_1_pr_19_p0_olifant_gebaren_video_raw.avi	f:\flesh_data_processed\01_XDF_processing\Data...	10_1_pr_19_p0_olifant_gebaren	True	f:\flesh_data_processed\01_XDF_processing\Data...
4	10_1_pr_1_p0_knippen_combinatie_video_raw.avi	f:\flesh_data_processed\01_XDF_processing\Data...	10_1_pr_1_p0_knippen_combinatie	True	f:\flesh_data_processed\01_XDF_processing\Data...
...	...	...	...	...	...
10375	9_2_trial_94_p0_ziek_gebaren_c0_video_raw.avi	f:\flesh_data_processed\01_XDF_processing\Data...	9_2_trial_94_p0_ziek_gebaren_c0	True	f:\flesh_data_processed\01_XDF_processing\Data...
10376	9_2_trial_95_p0_piepen_gebaren_c0_cut1_video_r...	f:\flesh_data_processed\01_XDF_processing\Data...	9_2_trial_95_p0_piepen_gebaren_c0_cut1	True	f:\flesh_data_processed\01_XDF_processing\Data...
10377	9_2_trial_96_p0_piepen_gebaren_c1_video_raw.avi	f:\flesh_data_processed\01_XDF_processing\Data...	9_2_trial_96_p0_piepen_gebaren_c1	True	f:\flesh_data_processed\01_XDF_processing\Data...
10378	9_2_trial_97_p0_piepen_gebaren_c2_cut1_video_r...	f:\flesh_data_processed\01_XDF_processing\Data...	9_2_trial_97_p0_piepen_gebaren_c2_cut1	True	f:\flesh_data_processed\01_XDF_processing\Data...
10379	9_2_trial_9_p0_kind_geluiden_c1_cut1_video_raw...	f:\flesh_data_processed\01_XDF_processing\Data...	9_2_trial_9_p0_kind_geluiden_c1_cut1	True	f:\flesh_data_processed\01_XDF_processing\Data...

10380 rows × 5 columns


Videos with MISSING corresponding webcam CSV:

	video_name	video_path	matching_key	csv_found	csv_path(s)

Final audio-video

Now we can do final combination of audio and video files. We use the same method as in XDF processing script

# Set folders
curfolder = os.getcwd()
# If folder data doesn't exist, create it
if not os.path.exists(os.path.join(curfolder, 'data')):
    os.makedirs(os.path.join(curfolder, 'data'))
datafolder = os.path.join(curfolder, 'data')

videofolder = trialfolder
audiovideo = os.path.join(datafolder, 'Data_processed', 'AudioVideo_final')
# check if it exists, if not create it
if not os.path.exists(os.path.join(datafolder, 'Data_processed', 'AudioVideo_final')):
    os.makedirs(os.path.join(datafolder, 'Data_processed', 'AudioVideo_final'))

if not os.path.exists(audiovideo):
    os.makedirs(audiovideo)

audio_trials_48 = os.path.join(curfolder, 'data', 'Data_processed', 'Data_trials', 'Audio_48')
wavfiles48 = glob.glob(os.path.join(audio_trials_48, '*.wav'))

# Inititate error log
errorlog = []

with16 = ['15_1', '16_2']

# loop over Audio files
for file in wavfiles48:

    print('Now processing file '+file)

    filename = file.split(os.sep)[-1].split('.')[0]
    dyadIndex = filename.split('_')[0]   # this is dyad number
    partIndex = filename.split('_')[1]

    if 'rein' in filename:
        sessionIndex = dyadIndex + '_' + partIndex + '_rein'
    else:
        sessionIndex = dyadIndex + '_' + partIndex

    if sessionIndex in with16:
        continue

    if '_rein_' in filename: 
        trialIndex = filename.split('_')[4] # this is trial number
        participant = filename.split('_')[8]
        word = filename.split('_')[9]
        modality = filename.split('_')[10]
        
    else:
        trialIndex = filename.split('_')[3] # this is trial number
        participant = filename.split('_')[7] # this is participant 0/1
        word = filename.split('_')[8] # this is the word
        modality = filename.split('_')[9] # this is the modality

    # handle correction
    if 'c0' in file:
        correction = '_c0'
    elif 'c1' in file:
        correction = '_c1'
    elif 'c2' in file:
        correction = '_c2'
    else:
        correction = ''
    # trial type
    if '_pr_' in file:
        trialtype = 'pr'
    else:
        trialtype = 'trial'

    if 'cut' in file:
        # take the last element
        cut = filename.split('_')[-1]
        cut = '_' + cut
    else:
        cut = ''


    # This is the location where the video will be saved
    output_path = os.path.abspath(os.path.join(audiovideo, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}_final.avi"))
  
    if os.path.exists(output_path):
        print('Video already exists, skipping...')
        continue

    #load in the audio
    print('Loading the audio')
    audio_path = file
    if not os.path.exists(audio_path):
        print(f"Audio file not found: {audio_path}")

    # input the video with ffmpg
    input_audio = ffmpeg.input(audio_path)
    print(input_audio)
    #load in the video with matchich trialIndex and SessionIndex
    print('Loading the video')

    video_path = os.path.join(videofolder, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}{cut}_video_raw.avi")

    if not os.path.exists(video_path):
        print(f"Video file not found: {video_path}")

    input_video = ffmpeg.input(video_path)
    print(input_video)
    
    #combine the audio and video
    print('Combining audio and video')

    try:
        ffmpeg.concat(input_video, input_audio, v=1, a=1).output(
            output_path,
            vcodec='libx264',
            acodec='aac',
            video_bitrate='2M',         
            
            ).run(overwrite_output=True)
    except:
        print('Error in combining audio and video')
        errorlog.append(file)
        continue

# now the same but with 16kHz audio
wavfolder16 = os.path.join(curfolder, 'data', 'Data_processed', 'Data_trials', 'Audio')
wavfiles16 = glob.glob(os.path.join(wavfolder16, '*.wav'))

for file in wavfiles16:

    print('Now processing file '+file)

    filename = file.split(os.sep)[-1].split('.')[0]
    dyadIndex = filename.split('_')[0]   # this is dyad number
    partIndex = filename.split('_')[1]

    if 'rein' in filename:
        sessionIndex = dyadIndex + '_' + partIndex + '_rein'
    else:
        sessionIndex = dyadIndex + '_' + partIndex

    if sessionIndex not in with16:
        continue

    if '_rein_' in filename: 
        trialIndex = filename.split('_')[4] # this is trial number
        participant = filename.split('_')[8]
        word = filename.split('_')[9]
        modality = filename.split('_')[10]
        
    else:
        trialIndex = filename.split('_')[3] # this is trial number
        participant = filename.split('_')[7] # this is participant 0/1
        word = filename.split('_')[8] # this is the word
        modality = filename.split('_')[9] # this is the modality

    # handle correction
    if 'c0' in file:
        correction = '_c0'
    elif 'c1' in file:
        correction = '_c1'
    elif 'c2' in file:
        correction = '_c2'
    else:
        correction = ''
    # trial type
    if '_pr_' in file:
        trialtype = 'pr'
    else:
        trialtype = 'trial'

    if 'cut' in file:
        # take the last element
        cut = filename.split('_')[-1]
        cut = '_' + cut
    else:
        cut = ''

    # This is the location where the video will be saved
    output_path = os.path.abspath(os.path.join(audiovideo, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}_final.avi"))
  
    if os.path.exists(output_path):
        print('Video already exists, skipping...')
        continue

    #load in the audio
    print('Loading the audio')
    audio_path = file
    if not os.path.exists(audio_path):
        print(f"Audio file not found: {audio_path}")

    # input the video with ffmpg
    input_audio = ffmpeg.input(audio_path)
    print(input_audio)
    #load in the video with matchich trialIndex and SessionIndex
    print('Loading the video')

    video_path = os.path.join(videofolder, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}{cut}_video_raw.avi")

    if not os.path.exists(video_path):
        print(f"Video file not found: {video_path}")

    input_video = ffmpeg.input(video_path)
    print(input_video)
    
    #combine the audio and video
    print('Combining audio and video')

    try:
        ffmpeg.concat(input_video, input_audio, v=1, a=1).output(
            output_path,
            vcodec='libx264',
            acodec='aac',
            video_bitrate='2M',         
            
            ).run(overwrite_output=True)
    except:
        print('Error in combining audio and video')
        
        errorlog.append(file)
        continue

        
# Write the error log
errorlog = pd.DataFrame(errorlog, columns=['File'])
currendate = pd.to_datetime("today").strftime("%Y%m%d")
outfile = os.path.join(audiovideo, f'errorlog_combining_audio_video_{currendate}.csv')

errorlog.to_csv(outfile, index=False, header=False)
print('All done!')

Now we can process with motion tracking and further processing