diff --git a/discover_overlay/voice_overlay.py b/discover_overlay/voice_overlay.py
index 23fe463..19ba113 100644
--- a/discover_overlay/voice_overlay.py
+++ b/discover_overlay/voice_overlay.py
@@ -544,9 +544,11 @@ class VoiceOverlayWindow(OverlayWindow):
                 speaking = "speaking" in user and user["speaking"]
 
                 # Extend timer if mid-speaking
+                if self.highlight_self and self_user == user:
+                    continue
                 if speaking:
                     user['lastspoken'] = perf_counter()
-                if not speaking:
+                else:
                     grace = self.only_speaking_grace_period
 
                     if (