Exclusion II: speech presence

Overview

In this script, we will use Whisper to check each trial for speech presence. Despite the fact that the algorithm hallucinates quite often, it will still help us to reduce the number of trials we need to check for potential speech presence.

Code to prepare the environment 
import os
import glob
import pandas as pd
import numpy as np
import whisper
import librosa
import re

curfolder = os.getcwd()
cleaningfolder = os.path.join(curfolder, 'Cleaning')

# Here are our audio files
wavfolder = os.path.join(curfolder, "..", "01_XDF_processing", "data", "Data_processed", "Data_trials", "Audio_48")
wavfiles = glob.glob(os.path.join(wavfolder, "*.wav"))
print('Found {} wav files'.format(len(wavfiles)))
Found 10378 wav files

Speech detection using Whisper

Custom functions 
model = whisper.load_model("large")  

def transcribe_dutch_chunk(y, sr, model):
    """
    Transcribe an audio chunk using Whisper model.
    Args:
        y: numpy array of audio data (mono)
        sr: sample rate (unused, Whisper handles resampling)
        model: Whisper model instance
    Returns:
        Whisper transcription result dictionary
    """
    result = model.transcribe(
        y,
        language="nl",       # or language=None to let it auto-detect
        task="transcribe",
        fp16=False,
        no_speech_threshold=1.0,   # don't aggressively drop "no speech"
        logprob_threshold=-2.0     # accept lower-confidence segments
    )
    return result

def chunk_has_speech_very_sensitive(result, min_chars=2):
    """Check if audio chunk contains speech with sensitive detection.
    Args:
        result: Whisper transcription result dictionary
        min_chars: Minimum character threshold for speech detection
    Returns:
        tuple: (bool indicating speech presence, extracted text)
    """
    text = (result.get("text") or "").strip()
    # If Whisper produced *anything* with letters, call it speech
    if len(text) < min_chars:
        return False, text
    if not re.search(r"[A-Za-zÀ-ÿ]", text):
        return False, text
    return True, text
results_rows = []

for audio_path in wavfiles:

    wav_name = os.path.basename(audio_path)
    print(f"Processing whole file: {wav_name}")

    # 1) load entire audio
    y_full, sr = librosa.load(audio_path, sr=None, mono=True)

    # skip extremely short files if you want
    if len(y_full) < 0.1 * sr:
        print("  Skipping (too short)")
        continue

    # 2) run Whisper on the whole file
    result = transcribe_dutch_chunk(y_full, sr, model)

    # 3) decide if speech is present
    has_speech, text = chunk_has_speech_very_sensitive(result)

    if has_speech:
        print(f"  🚨 Speech in full file {wav_name}: {text}")
    else:
        print(f"  OK: no speech in full file {wav_name}")

    # store for later analysis
    results_rows.append({
        "filename": wav_name,
        "start_time": 0.0,
        "end_time": len(y_full) / sr,
        "has_speech": has_speech,
        "whisper_text": text
    })

speech_scan_full = pd.DataFrame(results_rows)
speech_scan_full.to_csv(os.path.join("Cleaning", "whisper_dutch_speech_scan_fullfiles.csv"), index=False)

Before we start manually checking all flagged files, we need to preprocess the resulting file

# load in the document
speech_scan_full = pd.read_csv(os.path.join("Cleaning", "whisper_dutch_speech_scan_fullfiles.csv"))

# keep only those that has_speech == True
scan_speech = speech_scan_full[speech_scan_full['has_speech'] == True]

# if there is everything capital, this is okay and we can exclude those
scan_speech = scan_speech[~scan_speech['whisper_text'].str.isupper()]

# save
scan_speech.to_csv(os.path.join("Cleaning", "control_for_speech.csv"), index=False)

print(len(scan_speech))
scan_speech.head(20)
1117
filename start_time end_time has_speech whisper_text
0 10_1_trial_11_Mic_nominal_srate48000_p1_vuur_c... 0.0 9.282875 True Wat is dat?
8 10_1_trial_21_Mic_nominal_srate48000_p0_piepen... 0.0 9.297125 True Ik heb het gevoel dat het een beetje te hard is.
10 10_1_trial_23_Mic_nominal_srate48000_p0_vangen... 0.0 10.808375 True Wat is dat? Wat is dat?
11 10_1_trial_24_Mic_nominal_srate48000_p0_kauwen... 0.0 7.720625 True Dank je wel.
18 10_1_trial_32_Mic_nominal_srate48000_p1_hond_g... 0.0 5.534000 True Dank u wel.
19 10_1_trial_33_Mic_nominal_srate48000_p1_ei_geb... 0.0 6.586938 True Dank u wel.
22 10_1_trial_38_Mic_nominal_srate48000_p0_kat_ge... 0.0 4.135562 True Hoi. Hoi.
23 10_1_trial_39_Mic_nominal_srate48000_p0_dood_g... 0.0 6.863062 True Ik heb een vraag.
30 10_1_trial_47_Mic_nominal_srate48000_p1_kind_g... 0.0 7.241875 True Om. Om. Om. Om.
34 10_1_trial_50_Mic_nominal_srate48000_p1_hoorn_... 0.0 5.986750 True Tuk-tuk-tuk. Tuk-tuk-tuk.
39 10_1_trial_6_Mic_nominal_srate48000_p0_gooien_... 0.0 6.919937 True Wat?
40 10_1_trial_7_Mic_nominal_srate48000_p0_water_c... 0.0 11.878562 True Ik ben er niet voor. Dank je wel.
41 10_1_trial_8_Mic_nominal_srate48000_p0_wind_co... 0.0 6.618500 True Deze is een van de meeste uitgebreide vrachtwa...
43 10_2_trial_101_Mic_nominal_srate48000_p1_gromm... 0.0 6.300000 True Nu is het tijd om te beginnen.
53 10_2_trial_16_Mic_nominal_srate48000_p0_rennen... 0.0 4.844313 True Dank u wel.
56 10_2_trial_23_Mic_nominal_srate48000_p1_zwaar_... 0.0 6.969125 True Dank u wel.
66 10_2_trial_33_Mic_nominal_srate48000_p1_vallen... 0.0 8.376000 True Dank u wel.
67 10_2_trial_34_Mic_nominal_srate48000_p1_vallen... 0.0 10.216812 True Dank u wel. ***
74 10_2_trial_42_Mic_nominal_srate48000_p0_slaan_... 0.0 5.658750 True Dank u wel.
84 10_2_trial_53_Mic_nominal_srate48000_p1_luidru... 0.0 5.694500 True Uw man moet graag naar toe, de vrouw moet daar...

Processing annotated file

After manually inspecting all sound files and annotating whether there is speech or not, we now want to clean the annotated file so that only clear samples of speech use are left

# load the csv file
df = pd.read_csv(os.path.join(cleaningfolder, 'control_for_speech_check.csv'))
df.head()
filename check start_time end_time has_speech whisper_text
0 10_1_trial_11_Mic_nominal_srate48000_p1_vuur_c... no speech 0.0 9.282875 True Wat is dat?
1 10_1_trial_21_Mic_nominal_srate48000_p0_piepen... no sound 0.0 9.297125 True Ik heb het gevoel dat het een beetje te hard is.
2 10_1_trial_23_Mic_nominal_srate48000_p0_vangen... no sound 0.0 10.808375 True Wat is dat? Wat is dat?
3 10_1_trial_24_Mic_nominal_srate48000_p0_kauwen... no sound 0.0 7.720625 True Dank je wel.
4 10_1_trial_32_Mic_nominal_srate48000_p1_hond_g... no sound 0.0 5534.000000 True Dank u wel. 
# get rid of rows where check == no speech
df = df[df['check'] != 'no speech']

# get rid of rows where check == no sound
df = df[df['check'] != 'no sound']

# get rid of rows that contain string noisy or onisy
df = df[~df['check'].str.contains('noisy', case=False, na=False)]
df = df[~df['check'].str.contains('onisy', case=False, na=False)]

# get rid of rows that contain laugh or Laugh string
df = df[~df['check'].str.contains('laugh', case=False, na=False)]


df.head(15)
filename check start_time end_time has_speech whisper_text
159 19_1_trial_21_Mic_nominal_srate48000_p0_vrouw_... speech cut 0.0 5.699937 True Uw vrouw.
176 20_1_trial_16_Mic_nominal_srate48000_p1_kauwen... speech? Ja 0.0 9.825875 True Ik heb het gevoel dat ik een vrouw ben.
177 20_1_trial_17_Mic_nominal_srate48000_p1_slaan_... speech? 0.0 5.242625 True Kut.
190 20_2_trial_42_Mic_nominal_srate48000_p1_spring... speech 0.0 10.173562 True Wat is dat? ***
197 20_2_trial_66_Mic_nominal_srate48000_p1_blij_c... speech 0.0 8.846187 True Ik ga even kijken.
224 24_1_trial_40_Mic_nominal_srate48000_p0_scherp... speech at the end? 0.0 5.606250 True Wat is dat?
230 24_2_trial_27_Mic_nominal_srate48000_p1_bitter... ? 0.0 2.255375 True Doe het goed.
256 26_2_trial_55_Mic_nominal_srate48000_p0_verbra... speech 0.0 9.645938 True Oud.
260 26_2_trial_6_Mic_nominal_srate48000_p0_kind_ge... speech? 0.0 6.538687 True Dank je wel voor het kijken.
264 27_1_trial_11_Mic_nominal_srate48000_p1_ik_com... speech cut 0.0 21.121625 True Dank u wel. *** ***
291 27_2_trial_95_Mic_nominal_srate48000_p1_stil_g... speech? 0.0 13.039062 True Mooi. ***
295 28_2_trial_33_Mic_nominal_srate48000_p0_bang_g... speech 0.0 3.037313 True Oh, oh, oh.
349 30_2_trial_103_Mic_nominal_srate48000_p1_staar... speech 0.0 13.763063 True Dank u wel. Yes. Yes. Yes.
354 30_2_trial_30_Mic_nominal_srate48000_p1_kind_g... speech? mama, papa 0.0 4.209750 True Mama. Papa.
355 30_2_trial_31_Mic_nominal_srate48000_p1_kind_g... speech? mama papa 0.0 7.981438 True Moema. Moema. 
# save this
df.to_csv(os.path.join(cleaningfolder, 'control_for_speech_check_cleaned.csv'), index=False)