Pre-Processing III: Correcting trials

Overview

The trials are meant to be delimited by button box markers that are triggered by participants locking their hand in the beginning and in the end of each performance. However, due to occasional errors (e.g., participants forgot to lock their hands), we need to check all trials, and correct the ones that are not properly delimited.

We inspected and annotated all trials manually using custom script (see here). The custom annotator creates a log file where we log whether beginning or end needs to be cut. We then use this log file to correct the trials in this notebook.

Note that video and audio is created in the same way as in XDF processing script

Note: For documentation purposes, we store the time adaptations are in /data/Data_processed/Data_trials/Handlock_error/all_cuts_differences.csv

# import packages
import os
import glob
import pandas as pd
import numpy as np
import wave, struct, random
from scipy.io import wavfile
import noisereduce as nr
import cv2
import csv
from numpy import tri
from collections import defaultdict
from torch import le
import ffmpeg
import re


# Set folders
curfolder = os.getcwd()

# Data folder
datafolder = os.path.join(curfolder, 'data')
if not os.path.exists(datafolder):
    os.makedirs(datafolder)

# Also error_logs
errorlogs = os.path.join(datafolder, 'error_logs')
if not os.path.exists(errorlogs):
    os.makedirs(errorlogs)

experiment_to_process = os.path.join(curfolder, '..', '00_raw')

# Also CsvDataTS_raw
outputfolder = os.path.join(datafolder, 'Data_processed', 'CsvDataTS_raw')
if not os.path.exists(outputfolder):
    os.makedirs(outputfolder)

# Also Data_trials
trialfolder = os.path.join(datafolder, 'Data_processed', 'Data_trials')
if not os.path.exists(trialfolder):
    os.makedirs(trialfolder)

# Get all the folders in the target folder
datafolders = glob.glob(os.path.join(experiment_to_process, '*'))

# Extract the folder IDs
datafolders_id = [x.split(os.sep)[-1] for x in datafolders]

# Here we store whole csvs for sessions
timeseries = os.path.join(curfolder, 'data', 'Data_processed', 'CsvDataTS_raw')

print(datafolders_id)

# Identify all xdf files and all the associated triallist info
xdffiles = []
trialdatas = []

for i in datafolders_id:
    file = glob.glob(os.path.join(experiment_to_process, i, '*.xdf'))
    trialfile = os.path.join(experiment_to_process, i, i + '_results' + '.csv')
    trialdatas.append(trialfile)
    xdffiles.extend(file)

# These are the xdf files we need to process
print(xdffiles[0:10])
print(trialdatas[0:10])

['0_1', '0_2', '10_1', '10_2', '11_1', '11_2', '12_1', '12_2', '13_1', '13_2', '14_1', '14_2', '15_1', '15_2', '16_1', '16_2', '17_1', '17_2', '18_1', '18_2', '19_1', '19_2', '1_1', '1_2', '20_1', '20_2', '21_1', '22_1', '22_2', '23_1', '23_2', '24_1', '24_2', '26_1', '26_2', '27_1', '27_2', '28_1', '28_2', '29_1', '29_2', '2_1', '2_2', '30_1', '30_2', '31_1', '31_2', '32_1', '32_2', '33_1', '33_2', '34_1', '34_2', '35_1', '35_2', '36_1', '36_2', '37_1', '37_2', '38_1', '38_2', '39_1', '39_2', '3_1', '3_2', '40_1', '40_2', '41_1', '41_2', '42_1', '42_2', '43_1', '43_2', '44_1', '44_2', '45_1', '45_2', '46_1', '46_2', '47_1', '47_2', '48_1', '48_2', '49_1', '49_2', '4_1', '4_2', '50_1', '50_2', '51_1', '51_2', '52_1', '52_2', '53_1', '53_2', '54_1', '54_2', '55_1', '55_2', '56_1', '56_2', '57_1', '57_2', '58_1', '58_2', '59_1', '59_2', '5_1', '5_2', '60_1', '60_2', '61_1', '61_2', '62_1', '62_2', '63_1', '63_2', '64_1', '64_2', '65_1', '65_2', '66_1', '66_2', '67_1', '67_2', '69_1', '69_2', '6_1', '6_2', '70_1', '70_2', '71_1', '71_2', '72_1', '72_2', '7_1', '7_2', '8_1', '8_2', '9_1', '9_2', 'data_errors.txt', 'Demographics_all', 'Expressibility', 'README.txt']
['f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\0_1\\session_S001.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\0_2\\session_S002.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\18_1\\18_1_session.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\18_2\\18_2_reinitiated-session.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\23_2\\23_2_session.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\31_2\\31_2_session.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\35_2\\35_2_session.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\36_1\\36_1_reinitiated-session.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\36_1\\36_1_session.xdf', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\42_1\\42_1_session.xdf']
['f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\0_1\\0_1_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\0_2\\0_2_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\10_1\\10_1_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\10_2\\10_2_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\11_1\\11_1_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\11_2\\11_2_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\12_1\\12_1_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\12_2\\12_2_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\13_1\\13_1_results.csv', 'f:\\FLESH_ContinuousBodilyEffort\\01_XDF_processing\\..\\00_raw\\13_2\\13_2_results.csv']

# Function to parse tokens from file name
def parse_tokens(fname):
    """
    Parses a filename to extract metadata tokens.

    Parameters:
    -----------
    fname : str
        The filename to parse, typically in the format:
        - For 'rein' files: 'session_part_rein_trial_X_device_stream_srateX_pX_concept_cX.csv'
        - For non-'rein' files: 'session_part_trial_X_device_stream_srateX_pX_concept_cX.csv'

    Returns:
    --------
    dict
        A dictionary containing the extracted metadata with keys:
        - 'sessionID': The session identifier (e.g., '49_2_rein' or '49_2')
        - 'part': The participant identifier (e.g., '2')
        - 'trial': The trial number (e.g., '25')
        - 'pcn': The participant-concept number (e.g., 'p1')
        - 'concept': The concept name (e.g., 'bitter_geluiden')
        - 'correction': The correction token (e.g., 'c1')
    """

    b = os.path.basename(fname)
    tokens = b.split('_')
    if 'rein' in b:
        if 'BalanceBoard' in b:
            sessionID = '_'.join(tokens[0:3])
            part = tokens[1]
            trial = tokens[4] if len(tokens) > 4 else ''
            pcn = tokens[9] if len(tokens) > 7 else ''
            concept = tokens[10] if len(tokens) > 8 else ''
            
        if 'Mic' in b or 'MyWebcamFrameStream' in b:
            sessionID = '_'.join(tokens[0:3])
            part = tokens[1]
            trial = tokens[4] if len(tokens) > 4 else ''
            pcn = tokens[8] if len(tokens) > 7 else ''
            concept = tokens[9] if len(tokens) > 8 else ''
        else:
            sessionID = '_'.join(tokens[0:3])
            part = tokens[1] if len(tokens) > 1 else ''
            trial = tokens[4] if len(tokens) > 3 else ''
            pcn = tokens[5] if len(tokens) > 4 else ''
            concept = tokens[6] if len(tokens) > 5 else ''
            
    else: 
        sessionID = '_'.join(tokens[0:2])
        part = tokens[1] if len(tokens) > 1 else ''
        trial = tokens[3] if len(tokens) > 3 else ''
        pcn = tokens[7] if len(tokens) > 6 else ''
        concept = tokens[8] if len(tokens) > 7 else ''

    # Find correction token if present
    corr = ''
    for c in ('c0','c1','c2'):
        if c in b:
            corr = c
            break
    return {
        'sessionID': sessionID,
        'part': part,
        'trial': trial,
        'pcn': pcn,
        'concept': concept,
        'correction': corr
    }


# audio write function
def to_audio(fileloc, timeseries, samplerate = 16000, channels = 1):
    """
    Writes a timeseries to an audio file.

    Parameters:
    -----------
    fileloc : str
        Path to the output audio file.
    timeseries : list or array-like
        The audio data to be written.
    samplerate : int, optional
        Sampling rate of the audio (default is 16000).
    channels : int, optional
        Number of audio channels (default is 1 for mono).
    """

    obj = wave.open(fileloc,'w')
    obj.setnchannels(channels) # mono
    obj.setsampwidth(2)
    obj.setframerate(float(samplerate))
    for i in timeseries:
        data = struct.pack('<h', int(i))
        obj.writeframesraw( data )
    obj.close()
    
def write_video(vidloc, fourcc, originalfps, frameWidth, frameHeight, capture, frames):
    """
    Writes frames to a video file.

    Parameters:
    -----------
    vidloc : str
        Path to the output video file.
    fourcc : int or str
        FourCC code for the video codec (e.g., cv2.VideoWriter_fourcc(*'XVID')).
    originalfps : float
        Frames per second of the output video.
    frameWidth : int
        Width of each frame in the video.
    frameHeight : int
        Height of each frame in the video.
    capture : cv2.VideoCapture
        Video capture object to read frames from.
    frames : list or array-like
        List of frame indices to write to the video.
    """

    out = cv2.VideoWriter(vidloc, fourcc, fps = originalfps, frameSize = (int(frameWidth), int(frameHeight)))

    print('Looping over frames')

    for fra in frames:
        capture.set(cv2.CAP_PROP_POS_FRAMES, fra)
        ret, frame = capture.read()
        if ret:
            out.write(frame)
            
        if not ret:
            print('a frame was dropped: ' + str(fra))
        
    capture.release()
    out.release()

def find_closest_rows(df, target_number):
    """
    Finds the row(s) in a DataFrame whose first column value is closest to a target number.
    Returns a DataFrame containing the row(s) with the smallest absolute difference.
    """
    # Calculate the absolute difference between each value in the 'trial_start' column and the target number
    df['abs_difference'] = abs(df.iloc[:, 0] - target_number)
    
    # Find the row with the smallest absolute difference for each row
    closest_rows = df[df['abs_difference'] == df['abs_difference'].min()]

    return closest_rows

Preparing original files to cut

We first need to find again original files and corresponding information about trials.

# Define cut
errorfolder = os.path.join(trialfolder, 'Handlock_error') # here we store all the corrected frame ranges
errorfiles = glob.glob(os.path.join(errorfolder, '*.csv'))

cut_nr = 'cut26' # Specify the round nr.

# Keep only those that have cut_nr
cutfiles = [x for x in errorfiles if cut_nr in x]
print('There are ' + str(len(cutfiles)) + ' files with ' + cut_nr + ' in the name')

error_df = pd.read_csv(cutfiles[0])
trialids = []

# Loop over the rows and get elements 0:3, if there is rein the name, then 0:4
for i in range(len(error_df)):
    if 'rein' in error_df['FILE'][i]:
        trialids.append('_'.join(error_df['FILE'][i].split('_')[0:5]))
    else:
        trialids.append('_'.join(error_df['FILE'][i].split('_')[0:4]))

# Add _ to each in the list
trialids = [x + '_' for x in trialids]
print(trialids)
print(len(trialids))

# Build index of csv files 
csvfiles = glob.glob(os.path.join(trialfolder, '*.csv'))

# Get rid of srate48000
csvfiles = [x for x in csvfiles if 'srate48000' not in x]

# Keep only those in the trialids list
csvfiles = [x for x in csvfiles if any(tid in x for tid in trialids)]

index = defaultdict(lambda: {'audio': None, 'video': None, 'bb': None})
ts_cache = {}  # {csv_path: (first_ts, last_ts)}

print( f'Found {len(csvfiles)} CSV files in {trialfolder}.')

for csv_file in csvfiles:
    base = os.path.basename(csv_file)
    tokens = parse_tokens(base)
    key = (tokens['sessionID'], (tokens['concept'], tokens['correction'])) 
    # Decide stream and which column to read (header=None to use iloc indices)
    if 'Mic' in base:
        #print('this is audio')
        device = 'audio'; usecols = [0]
    elif 'MyWebcamFrameStream' in base:
        #print('this is video')
        device = 'video'; usecols = [0]
    elif 'BalanceBoard' in base:
        #print('this is balance board')
        device = 'bb'; usecols = [0]

    else:
        continue

    # Store file path (prefer first seen)
    if index[key][device] is None:
        index[key][device] = csv_file

    # Read minimal data for timestamps once
    if csv_file not in ts_cache:
        try:
            # First line is header
            ts = pd.read_csv(csv_file, usecols=usecols)
            first_ts = ts.iloc[0, 0]
            last_ts = ts.iloc[len(ts) - 1, 0]
            ts_cache[csv_file] = (first_ts, last_ts)
        except Exception as e:
            # If reading fails, store None
            print(f'Failed to read timestamps from {csv_file}: {e}')
            ts_cache[csv_file] = (None, None)

print(f'Built index with {len(index)} unique keys and {len(ts_cache)} timestamp caches.')

Found 12 CSV files in f:\flesh_data_processed\01_XDF_processing\data\Data_processed\Data_trials.
Built index with 2 unique keys and 12 timestamp caches.

Now we add to the log file information about which original file corresponds to which trial, so that we can later cut the trials from the original files.

# Leave in error files only those that have cutX in the name
errorfiles = [x for x in errorfiles if cut_nr in x]
print(errorfiles)

['f:\\flesh_data_processed\\01_XDF_processing\\data\\Data_processed\\Data_trials\\Handlock_error\\cut26.csv']

# Build session-based map once
session_map = defaultdict(list)
for p in csvfiles:
    base = os.path.basename(p)
    toks = base.split('_')
    if len(toks) >= 2:
        key = (toks[0], toks[1])
        session_map[key].append(p)
    else:
        session_map[('','')].append(p)  # fallback bucket

# Helper: safe token getter
def tok_at(tokens, idx):
    try:
        return tokens[idx]
    except Exception:
        return ''

# Device tag checks (lowercase)
def is_audio_basename(b):
    lb = b.lower()
    return 'mic' in lb or 'microphone' in lb

def is_video_basename(b):
    lb = b.lower()
    return 'webcam' in lb or 'mywebcamframestream' in lb

def is_bb_basename(b):
    lb = b.lower()
    return 'balanceboard' in lb or 'balance_board' in lb

# Main per-file loop
for file in errorfiles:
    print('Processing error file:', file)
    error = pd.read_csv(file)

    # Detect filename column
    if 'FILE' in error.columns:
        fname_col = 'FILE'
    elif 'filename' in error.columns:
        fname_col = 'filename'
    else:
        obj_cols = error.select_dtypes(include=['object']).columns.tolist()
        fname_col = obj_cols[0] if obj_cols else error.columns[0]

    # Ensure columns exist
    for c in ['audio', 'video', 'bb',
              'audio_begin_time', 'audio_end_time',
              'video_begin_time', 'video_end_time',
              'bb_begin_time', 'bb_end_time',
              'sessionID', 'part', 'trial', 'pcn', 'concept', 'correction']:
        if c not in error.columns:
            error[c] = ''

    # Parse tokens for all filenames (keeps previous parse_tokens usage)
    parsed = error[fname_col].astype(str).apply(parse_tokens)
    parsed_df = pd.DataFrame(list(parsed))
    for col in ['sessionID', 'part', 'trial', 'pcn', 'concept', 'correction']:
        error[col] = parsed_df[col]

    # Iterate rows
    for i, row in error.iterrows():
        file_basename = os.path.basename(str(row[fname_col]))
        file_tokens = file_basename.split('_')
        sessionID = str(row['sessionID']) if pd.notna(row['sessionID']) else ''
        concept = str(row['concept']) if pd.notna(row['concept']) else ''
        #print('concept:', concept)
        correction = str(row['correction']) if pd.notna(row['correction']) else ''
        
        # Positional tokens for strict positional checks
        f_tok0 = tok_at(file_tokens, 0).lower()
        f_tok1 = tok_at(file_tokens, 1).lower()
        if 'rein' in file_basename:
            f_tok3 = tok_at(file_tokens, 4).lower()
        else:
            f_tok3 = tok_at(file_tokens, 3).lower()

        # Narrow candidates by session_map (first two tokens)
        key = (f_tok0, f_tok1)
        candidates = session_map.get(key, [])
        #print('candidates:', candidates)
        if not candidates:
            # fallback: try scanning all csvfiles
            candidates = csvfiles

        # Collect best matches per stream
        best_audio = None
        best_video = None
        best_bb = None

        for csv_path in candidates:
            base = os.path.basename(csv_path)
            csv_tokens = base.split('_')
            #print(csv_tokens)
            c_tok0 = tok_at(csv_tokens, 0).lower()
            c_tok1 = tok_at(csv_tokens, 1).lower()
            if 'rein' in base:
                c_tok3 = tok_at(csv_tokens, 4).lower()
            else:
                c_tok3 = tok_at(csv_tokens, 3).lower()
            lb = base.lower()

            # Check positional matches: first, second, fourth (if present)
            if not (f_tok0 == c_tok0 and f_tok1 == c_tok1):
                continue
            # If both have a 4th token, require they match (otherwise allow)
            if f_tok3 and c_tok3 and f_tok3 != c_tok3:
                # mismatch on 4th token -> skip
                continue

            # Concept and correction must be present in CSV basename
            if concept and concept.lower() not in lb:
                continue
            if correction and correction.lower() not in lb:
                continue

            # Now decide stream
            if is_audio_basename(base):
                # Prefer exact matches that also include p0/p1/trial/pcn if available
                best_audio = best_audio or csv_path
                # If we find one that also contains trial/pcn, prefer it
                if (str(row['trial']) and str(row['trial']) in base) or (str(row['pcn']) and str(row['pcn']) in base):
                    best_audio = csv_path
            elif is_video_basename(base):
                best_video = best_video or csv_path
                if (str(row['trial']) and str(row['trial']) in base) or (str(row['pcn']) and str(row['pcn']) in base):
                    best_video = csv_path
            elif is_bb_basename(base):
                best_bb = best_bb or csv_path
                if (str(row['trial']) and str(row['trial']) in base) or (str(row['pcn']) and str(row['pcn']) in base):
                    best_bb = csv_path

        # Write found candidates into dataframe (use ts_cache for timestamps)
        def write_device(path, device):
            if not path:
                return
            error.at[i, device] = path
            first_ts, last_ts = ts_cache.get(path, (None, None))
            if device == 'audio':
                error.at[i, 'audio_begin_time'] = first_ts
                error.at[i, 'audio_end_time'] = last_ts
            elif device == 'video':
                error.at[i, 'video_begin_time'] = first_ts
                error.at[i, 'video_end_time'] = last_ts
            elif device == 'bb':
                error.at[i, 'bb_begin_time'] = first_ts
                error.at[i, 'bb_end_time'] = last_ts

        write_device(best_audio, 'audio')
        write_device(best_video, 'video')
        write_device(best_bb, 'bb')

    # Save updated error file
    error.to_csv(file, index=False)
    print('Saved', file)

Processing error file: f:\flesh_data_processed\01_XDF_processing\data\Data_processed\Data_trials\Handlock_error\cut26.csv
Saved f:\flesh_data_processed\01_XDF_processing\data\Data_processed\Data_trials\Handlock_error\cut26.csv

# Load it back in
error = pd.read_csv(errorfiles[0])
error.head()

	FILE	NEWBEGIN	NEWEND	audio	video	bb	audio_begin_time	audio_end_time	video_begin_time	video_end_time	bb_begin_time	bb_end_time	sessionID	part	trial	pcn	concept	correction
0	18_2_rein_trial_68_p1_horen_geluiden_c1_cut25_...	-2	-1	f:\flesh_data_processed\01_XDF_processing\data...	f:\flesh_data_processed\01_XDF_processing\data...	f:\flesh_data_processed\01_XDF_processing\data...	20660.190368	20664.76869	20660.164669	20664.741144	20660.089503	20664.66399	18_2_rein	2	68	p1	horen	c1

# Check if there are any nans
nans = error[error['video'].isna()]
print(nans)

Empty DataFrame
Columns: [FILE, NEWBEGIN, NEWEND, audio, video, bb, audio_begin_time, audio_end_time, video_begin_time, video_end_time, bb_begin_time, bb_end_time, sessionID, part, trial, pcn, concept, correction]
Index: []

Recutting files

Now we have link to the original files and we can create new csv files by extending/shortening them by a certain constant. The constant is either selected for all (this was done during the first annotating rounds), or individually per each file (this was done in later annotations when defining individual constants was more feasible)

Note that we need to manually update the newcut_nr

from hmac import new
from requests import session

newcut_nr = 'cut26' # UPDATE MANUALLY

csv_ts = glob.glob(os.path.join(timeseries, '*.csv'))

webcam_ts = [x for x in csv_ts if 'Webcam' in x]
mic_ts = [x for x in csv_ts if 'Mic_nominal' in x]
bb_ts = [x for x in csv_ts if 'BalanceBoard' in x]

for file in cutfiles:

    #### For files where we cut the beginning and end by a constant ####
    # beginlonger = False
    # endlonger = False
    # beginshorter = False
    # endshorter = False

    
    # if 'beginlonger' in file:
    #     beginlonger = True

    # if 'endlonger' in file:
    #     endlonger = True

    # if 'beginshorter' in file:
    #     beginshorter = True

    # if 'endshorter' in file:
    #     endshorter = True

    # if 'bothlonger' in file:
    #     beginlonger = True
    #     endlonger = True

    # if 'bothshorter' in file:
    #     beginshorter = True
    #     endshorter = True

    # constant = 0.6


    ###################################

    error = pd.read_csv(file)

    # Loop over the error words
    for i in range(len(error)):
        print('Now processing row: ' + str(i) + ' of file: ' + file)

        sessionID = error['sessionID'][i] 
        
        webcam_ts = os.path.join(timeseries, sessionID + '_MyWebcamFrameStream_nominal_srate500.csv')
        ts_w = pd.read_csv(webcam_ts)
        mic_ts = os.path.join(timeseries, sessionID + '_Mic_nominal_srate16000.csv')
        ts_m = pd.read_csv(mic_ts)
        bb_ts = os.path.join(timeseries, sessionID + '_BalanceBoard_stream_nominal_srate500.csv')
        ts_bb = pd.read_csv(bb_ts)

        ###### For files where we specified how much it should be cut 
        constant_beg = error['NEWBEGIN'][i]
        constant_end = error['NEWEND'][i]

        ###################################

        # Video correction 
        begintime = error['video_begin_time'][i]
        endtime = error['video_end_time'][i]
        new_begintime = begintime
        new_endtime = endtime

        # Add the constants to begin and end
        new_begintime = begintime + constant_beg
        new_endtime = endtime + constant_end

        ##### For files 
        # if beginlonger:
        #     new_begintime = begintime - constant

        # if beginshorter:
        #     new_begintime = begintime + constant*1.5

        # if endlonger:
        #     new_endtime = endtime + constant*3

        # if endshorter:
        #     new_endtime = endtime - constant*3

        # cut the new times from the ts_w, do not use function
        indices = (ts_w.iloc[:,0] >= new_begintime) & (ts_w.iloc[:,0] <= new_endtime)
        subset = ts_w.loc[indices, :]

        # save the new ts using the same path as the video with extension _cut1
        filename = error['video'][i]
        # replace .csv by _corrected.csv
        new_filename = filename.replace('_cut25.csv', '_cut26.csv')
        pd.DataFrame(subset).to_csv(new_filename, index=False)

        # save also newtime in the error dataframe
        error.loc[i, 'video_new_begin_time'] = new_begintime
        error.loc[i, 'video_new_end_time'] = new_endtime

        # audio correction
        begintime = error['audio_begin_time'][i]
        endtime = error['audio_end_time'][i]

        new_begintime = begintime
        new_endtime = endtime

        # add the constants to begin and end
        new_begintime = begintime + constant_beg
        new_endtime = endtime + constant_end

        #### For files where we cut the beginning and end by a constant ####

        # if beginlonger:
        #     new_begintime = begintime - constant

        # if beginshorter:
        #     new_begintime = begintime + constant*1.5

        # if endlonger:
        #     new_endtime = endtime + constant*3

        # if endshorter:
        #     new_endtime = endtime - constant*3

        ###################################

        # Cut the new times from the ts_m
        indices = (ts_m.iloc[:,0] >= new_begintime) & (ts_m.iloc[:,0] <= new_endtime)
        mic_subset = ts_m.loc[indices, :]

        # Save the new ts using the same path as the audio with extension _cutX
        file_name = error['audio'][i]
        new_filename = file_name.replace('_cut25.csv', '_cut26.csv') # UPDATE MANUALLY
        pd.DataFrame(mic_subset).to_csv(new_filename, index=False)

        # Save also newtime in the error dataframe
        error.loc[i, 'audio_new_begin_time'] = new_begintime
        error.loc[i, 'audio_new_end_time'] = new_endtime

        # Balanceboard correction
        begintime = error['bb_begin_time'][i]
        endtime = error['bb_end_time'][i]

        new_begintime = begintime
        new_endtime = endtime

        # Add the constants to begin and end
        new_begintime = begintime + constant_beg
        new_endtime = endtime + constant_end

        #### For files where we cut the beginning and end by a constant ####

        # if beginlonger:
        #     new_begintime = begintime - constant

        # if beginshorter:
        #     new_begintime = begintime + constant*1.5

        # if endlonger:
        #     new_endtime = endtime + constant*3

        # if endshorter:
        #     new_endtime = endtime - constant*3

        #######################################

        # Cut the new times from the ts_bb
        indices = (ts_bb.iloc[:,0] >= new_begintime) & (ts_bb.iloc[:,0] <= new_endtime)
        bb_subset = ts_bb.loc[indices, :]

        # Save the new ts using the same path as the balanceboard with extension _cutX
        file_name = error['bb'][i]
        new_filename = file_name.replace('_cut25.csv', '_cut26.csv') ## UPDATE MANUALLY
        pd.DataFrame(bb_subset).to_csv(new_filename, index=False)

        # Save also newtime in the error dataframe
        error.loc[i, 'bb_new_begin_time'] = new_begintime
        error.loc[i, 'bb_new_end_time'] = new_endtime

Now processing row: 0 of file: f:\flesh_data_processed\01_XDF_processing\data\Data_processed\Data_trials\Handlock_error\cut26.csv

Creating new video and audio

Now we can use the corrected files to create new wav files and video files, in the same way we did in XDF processing script

# Get all the csv files with cutX in it
correct = glob.glob(os.path.join(trialfolder, '*cut26*.csv')) # UPDATE MANUALLY
audio = os.path.join(trialfolder, 'Audio')

for i in range(len(correct)):
    #print(correct[i])
    sessionID = correct[i].split(os.sep)[-1].split('_')[0] + '_' + correct[i].split(os.sep)[-1].split('_')[1]

    # If it has Mic in it, create an audio
    if 'Mic' in correct[i]:
        print('This is an audio')
        file_name = correct[i]
        file_name = file_name.split(os.sep)[-1]
        file_name = file_name.replace('.csv', '.wav')          
        wavloc = os.path.join(audio, file_name)

        if os.path.exists(wavloc):
            continue
  
        print(file_name)
        ts = pd.read_csv(correct[i])
        sound = ts.iloc[:,1].tolist()
        to_audio(wavloc, sound)
        # Load data    
        rate, data = wavfile.read(wavloc)
        # Perform noise reduction
        reduced_noise = nr.reduce_noise(y=data, sr=rate, n_std_thresh_stationary=1.5,stationary=True)
        file_name2 = file_name.replace('.wav', '_denoised.wav')
        wavloc2 = os.path.join(audio, file_name2)
        wavfile.write(wavloc2, rate, reduced_noise)

    elif 'Webcam' in correct[i]:
        print('This is a video')

        filename = correct[i].split(os.sep)[-1]
        filename = filename.replace('MyWebcamFrameStream_nominal_srate500_', '')
        filename = filename.replace('.csv', '')
        vidloc = os.path.join(trialfolder, filename + '_video_raw' + '.avi')

        if os.path.exists(vidloc):
            continue

        ts = pd.read_csv(correct[i])
        if 'rein' in correct[i]:
            sessionID = sessionID.split('_')[0] + '_' + sessionID.split('_')[1] 
            videolong = os.path.join(experiment_to_process, sessionID, sessionID + '_reinitiated-video.avi')
        else:
            videolong = os.path.join(experiment_to_process, sessionID, sessionID + '-video.avi')
            
        begin_time = ts['0'].min() # begin time of the trial
        end_time = ts['0'].max() # end time of the trial
        # Get the begin and end frame
        begin_frame = ts['1'].min()
        end_frame = ts['1'].max()
        begin_frame = int(begin_frame)
        end_frame = int(end_frame)
        totframes = end_frame-begin_frame # total number of frames in the trial
        frames = range(begin_frame, end_frame) #get all the frames in trial
        
        # Load in the long video
        print('Loading the original video')
        capture = cv2.VideoCapture(videolong) 
        originalfps = round((totframes/(end_time-begin_time)),3)
        print('original fps: '+str(originalfps))
        
        # Update filename
        filename = correct[i].split(os.sep)[-1]
        filename = filename.replace('MyWebcamFrameStream_nominal_srate500_', '')
        filename = filename.replace('.csv', '')
        vidloc = os.path.join(trialfolder, filename + '_video_raw' + '.avi')

        # Video metadata
        fourcc = cv2.VideoWriter_fourcc(*'XVID')
        frameWidth = capture.get(cv2.CAP_PROP_FRAME_WIDTH)
        frameHeight = capture.get(cv2.CAP_PROP_FRAME_HEIGHT)
        
        # Start writing video
        print('Starting to write the video')
        write_video(vidloc, fourcc, originalfps, frameWidth, frameHeight, capture, frames)

    else:
        print('BB do not need any processing')

print('All streams have been corrected')

Align again with 48kHz audio

Now we also need to again align the newly cuted trials with the 48 kHz audio as we did in Audiosync script

# Collect all the files that have cutX in name
newcutfiles = glob.glob(os.path.join(trialfolder, '*cut26*.csv')) # UDPATE MANUALLY

sessionids = []
for f in newcutfiles:
    if 'rein' in f:
        sessionid = f.split(os.sep)[-1].split('_')[0] + '_' + f.split(os.sep)[-1].split('_')[1] + '_rein'
    else:
        sessionid = f.split(os.sep)[-1].split('_')[0] + '_' + f.split(os.sep)[-1].split('_')[1]
    
    if sessionid not in sessionids:
        sessionids.append(sessionid)

audio_48_files = glob.glob(os.path.join(experiment_to_process, '**', '*aligned*denoised*.wav'), recursive=True)

# Collect all 16kHz session files
ts_audio = os.path.join(curfolder, 'data', 'Data_processed', 'CsvDataTS_raw')
csvfiles = glob.glob(os.path.join(ts_audio, '*.csv'))
ts_audio_files = [x for x in csvfiles if 'Mic' in x]

# Collect all 16 kHz trial files
trials = os.path.join(curfolder, 'data', 'Data_processed', 'Data_trials')
csvfiles = glob.glob(os.path.join(trials, '*.csv'))
ts_audio_trials = [x for x in csvfiles if 'Mic' in x]

# Keep only those that have cut26 in name
ts_audio_trials = [x for x in ts_audio_trials if 'cut26' in x] # UPDATE MANUALLY

# Here we will store the new audio
audio_trials_48 = os.path.join(curfolder, 'data', 'Data_processed', 'Data_trials', 'Audio_48')
if not os.path.exists(audio_trials_48):
    os.makedirs(audio_trials_48)

error_list = []
convert = 3 # 48000/16000

# These have no 48 kHz
no48 = ['15_1', '16_2']

for file in ts_audio_files:
    print('working on ' + file)

    # Get session ID
    if 'rein' in file:
        sessionID = file.split(os.sep)[-1].split('.')[0].split('_')[0] + '_' + file.split(os.sep)[-1].split('.')[0].split('_')[1] + '_rein'
    else:
        sessionID = file.split(os.sep)[-1].split('.')[0].split('_')[0] + '_' + file.split(os.sep)[-1].split('.')[0].split('_')[1]


    if sessionID not in sessionids:
        print('sessionID ' + sessionID + ' not need to be reprocessed, skipping...')
        continue

    if sessionID in no48:
        print('sessionID ' + sessionID + ' has no 48kHz file, skipping...')
        continue

    # Find the corresponding 48kHz file
    try:
        if 'rein' in file:
            files48 = [x for x in audio_48_files if sessionID == (x.split(os.sep)[-1].split('.')[0].split('_')[0] + '_' + x.split(os.sep)[-1].split('.')[0].split('_')[1] + '_rein')]
            # Get rid of those that do NOT have rein in them
            files48 = [x for x in files48 if '_rein_' in x]
            file48 = files48[0]
        else:
            file48 = [x for x in audio_48_files if sessionID == (x.split(os.sep)[-1].split('.')[0].split('_')[0] + '_' + x.split(os.sep)[-1].split('.')[0].split('_')[1])]
            # Get rid of the one with _rein_
            file48 = [x for x in file48 if '_rein_' not in x][0]

        print('corresponding file: ' + file48)

    except:
        print('no corresponding file found')
        error_list.append('no corresponding file found for' + file)
        continue

    # Open audio
    audio48 = wave.open(file48, 'rb')
    data48 = np.frombuffer(audio48.readframes(-1), dtype=np.int16)

    # Open the csv file
    ts_audio = pd.read_csv(file)
    #print(ts_audio)

    # Get all the audiotrials_ids that have exactly identical sessionID
    if 'rein' in file:
        audiotrials = [
            x for x in ts_audio_trials 
            if sessionID == (x.split(os.sep)[-1].split('.')[0].split('_')[0] + '_' + x.split(os.sep)[-1].split('.')[0].split('_')[1] + '_rein')
        ]
        audiotrials = [x for x in audiotrials if '_rein_' in x]
    else:
        audiotrials = [
        x for x in ts_audio_trials 
        if sessionID == (x.split(os.sep)[-1].split('.')[0].split('_')[0] + '_' + x.split(os.sep)[-1].split('.')[0].split('_')[1])
    ]
        audiotrials = [x for x in audiotrials if '_rein_' not in x]

    print(audiotrials)

    for trial in audiotrials:
        print('working on trial: ' + trial)

        filename_new = trial.split(os.sep)[-1].split('.')[0]
        filename_new = filename_new.replace('16000', '48000')
        file_path = os.path.join(audio_trials_48, filename_new + '.csv')

        wavloc = os.path.join(audio_trials_48, filename_new + '.wav')

        if os.path.exists(wavloc):
            print(wavloc + ' already exists, skipping...')
            continue

        # Open the csv file
        df = pd.read_csv(trial)

        # Find first and last value
        start = df.iloc[0, 0]
        #print('start ', start)
        end = df.iloc[len(df)-1, 0]
        #print('end ', end)

        # Find the closest row in ts_audio
        closest_row_start = find_closest_rows(ts_audio, start)
        start_index = closest_row_start.index[0]    
        #print('start index ', start_index)
        closest_row_end = find_closest_rows(ts_audio, end)
        end_index = closest_row_end.index[0]
        #print('end index ', end_index)

        # Convert
        start_index_2 = start_index*convert
        end_index_2 = end_index*convert
        print('new indices ', start_index_2, end_index_2)

        # Cut the data48 and create the file
        data2_trial = data48[int(start_index_2):int(end_index_2)]
        to_audio(wavloc, data2_trial, samplerate = 48000)

    # Close the audio and clean the buffer
    audio48.close()
    audio48 = None
    data48 = None
    ts_audio = pd.DataFrame()

# Save the error list to a csv file
error_df = pd.DataFrame(error_list, columns=['error_files'])
error_df.to_csv(os.path.join(curfolder, 'data', 'Data_processed', 'Data_trials', 'Audio_48', 'error_files_audio48.csv'), index=False)

Combine audio and video

Now we combine the audio and video into one file, so we can re-check whether the correction helped. This is again same workflow as in Audiosync script

newcut_nr = 'cut26' # UPDATE MANUALLY

# Define the folders
videofolder = trialfolder

# Save them into separate folder
newfolder = 'AudioVideo_' + newcut_nr
audiovideo = os.path.join(datafolder, 'Data_processed', newfolder)

if not os.path.exists(audiovideo):
    os.makedirs(audiovideo)

wavfiles48 = glob.glob(os.path.join(audio_trials_48, '*.wav'))
wavfiles48 = [x for x in wavfiles48 if newcut_nr in x]

# Inititate error log
errorlog = []

# These sessions have no 48kHz audio
with16 = ['15_1', '16_2']

for file in wavfiles48:

    print('Now processing file '+ file)

    filename = file.split(os.sep)[-1].split('.')[0]
    dyadIndex = filename.split('_')[0]   # this is dyad number
    partIndex = filename.split('_')[1]

    if 'rein' in filename:
        sessionIndex = dyadIndex + '_' + partIndex + '_rein'
    else:
        sessionIndex = dyadIndex + '_' + partIndex

    if sessionIndex in with16:
        continue

    if '_rein_' in filename: 
        trialIndex = filename.split('_')[4] # this is trial number
        participant = filename.split('_')[8]
        word = filename.split('_')[9]
        modality = filename.split('_')[10]
        
    else:
        trialIndex = filename.split('_')[3] # this is trial number
        participant = filename.split('_')[7] # this is participant 0/1
        word = filename.split('_')[8] # this is the word
        modality = filename.split('_')[9] # this is the modality

    # Handle correction
    if 'c0' in file:
        correction = '_c0'
    elif 'c1' in file:
        correction = '_c1'
    elif 'c2' in file:
        correction = '_c2'
    else:
        correction = ''

    # Trial type
    if '_pr_' in file:
        trialtype = 'pr'
    else:
        trialtype = 'trial'

    # Cut
    if filename.split('_')[-2].startswith('cut'):
        add = '_' + filename.split('_')[-2]
    else:
        add = ''

    cut = '_' + newcut_nr

    # This is the location where the video will be saved
    output_path = os.path.abspath(os.path.join(audiovideo, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}{cut}_final.avi"))
  
    if os.path.exists(output_path):
        print('Video already exists, skipping...')
        continue

    # Load in the audio
    print('Loading the audio')
    audio_path = file
    if not os.path.exists(audio_path):
        print(f"Audio file not found: {audio_path}")

    input_audio = ffmpeg.input(audio_path)
    print(input_audio)
    
    # Load in the video with matching trialIndex and SessionIndex
    print('Loading the video')

    video_path = os.path.join(videofolder, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}{add}{cut}_video_raw.avi")

    if not os.path.exists(video_path):
        print(f"Video file not found: {video_path}")

    input_video = ffmpeg.input(video_path)
    print(input_video)
    
    # Combine the audio and video
    print('Combining audio and video')

    try:
        ffmpeg.concat(input_video, input_audio, v=1, a=1).output(
            output_path,
            vcodec='libx264',
            acodec='aac',
            video_bitrate='2M',         
            
            ).run(overwrite_output=True)
    except:
        print('Error in combining audio and video')
        errorlog.append(file)
        continue

# Now the same but with 16kHz audio
wavfolder16 = os.path.join(curfolder, 'data', 'Data_processed', 'Data_trials', 'Audio')
wavfiles16 = glob.glob(os.path.join(wavfolder16, '*.wav'))

# Keep only those that have cut1
wavfiles16 = [x for x in wavfiles16 if newcut_nr in x]

for file in wavfiles16:

    print('Now processing file '+file)

    filename = file.split(os.sep)[-1].split('.')[0]
    dyadIndex = filename.split('_')[0]   # this is dyad number
    partIndex = filename.split('_')[1]

    if 'rein' in filename:
        sessionIndex = dyadIndex + '_' + partIndex + '_rein'
    else:
        sessionIndex = dyadIndex + '_' + partIndex

    if sessionIndex not in with16:
        continue

    if '_rein_' in filename: 
        trialIndex = filename.split('_')[4] # this is trial number
        participant = filename.split('_')[8]
        word = filename.split('_')[9]
        modality = filename.split('_')[10]
        
    else:
        trialIndex = filename.split('_')[3] # this is trial number
        participant = filename.split('_')[7] # this is participant 0/1
        word = filename.split('_')[8] # this is the word
        modality = filename.split('_')[9] # this is the modality

    # Handle correction
    if 'c0' in file:
        correction = '_c0'
    elif 'c1' in file:
        correction = '_c1'
    elif 'c2' in file:
        correction = '_c2'
    else:
        correction = ''
    
    # Trial type
    if '_pr_' in file:
        trialtype = 'pr'
    else:
        trialtype = 'trial'

    # Handle cut 
    if filename.split('_')[-2].startswith('cut'):
        add = '_' + filename.split('_')[-2]
    else:
        add = ''

    # This is the location where the video will be saved
    output_path = os.path.abspath(os.path.join(audiovideo, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}{cut}_final.avi"))
  
    if os.path.exists(output_path):
        print('Video already exists, skipping...')
        continue

    # Load in the audio
    print('Loading the audio')
    audio_path = file
    if not os.path.exists(audio_path):
        print(f"Audio file not found: {audio_path}")

    input_audio = ffmpeg.input(audio_path)
    
    # Load in the video with matchich trialIndex and SessionIndex
    print('Loading the video')
    video_path = os.path.join(videofolder, f"{sessionIndex}_{trialtype}_{trialIndex}_{participant}_{word}_{modality}{correction}{add}{cut}_video_raw.avi")

    if not os.path.exists(video_path):
        print(f"Video file not found: {video_path}")

    input_video = ffmpeg.input(video_path)
    
    # Combine the audio and video
    print('Combining audio and video')

    try:
        ffmpeg.concat(input_video, input_audio, v=1, a=1).output(
            output_path,
            vcodec='libx264',
            acodec='aac',
            video_bitrate='2M',         
            
            ).run(overwrite_output=True)
    except:
        print('Error in combining audio and video')
        errorlog.append(file)
        continue

        
# Write the error log
errorlog = pd.DataFrame(errorlog)
errorlog.to_csv(os.path.join(audiovideo, 'errorlog_audio_video.csv'), index = False)
print('All done!')

If upon visual inspection we still need to cut further, we repeat the whole workflow with new document defining the corrected ranges. If all the files are correctly cut, we can progress with further processing