diff --git a/discover_overlay/discover_overlay.py b/discover_overlay/discover_overlay.py
index 98d803e..5b216d1 100755
--- a/discover_overlay/discover_overlay.py
+++ b/discover_overlay/discover_overlay.py
@@ -236,6 +236,8 @@ class Discover:
             "main", "square_avatar", fallback=True))
         self.voice_overlay.set_only_speaking(config.getboolean(
             "main", "only_speaking", fallback=False))
+        self.voice_overlay.set_only_speaking_grace_period(config.getint(
+            "main", "only_speaking_grace", fallback=0))
         self.voice_overlay.set_highlight_self(config.getboolean(
             "main", "highlight_self", fallback=False))
         self.voice_overlay.set_icon_only(config.getboolean(
diff --git a/discover_overlay/glade/settings.glade b/discover_overlay/glade/settings.glade
index a647255..65f8ef0 100644
--- a/discover_overlay/glade/settings.glade
+++ b/discover_overlay/glade/settings.glade
@@ -136,6 +136,13 @@
     <property name="step-increment">1</property>
     <property name="page-increment">8</property>
   </object>
+  <object class="GtkAdjustment" id="voice_display_speakers_grace_period_adj">
+    <property name="lower">0</property>
+    <property name="upper">360</property>
+    <property name="value">0</property>
+    <property name="step-increment">1</property>
+    <property name="page-increment">10</property>
+  </object>
   <object class="GtkAdjustment" id="voice_nick_lenght_adj">
     <property name="lower">10</property>
     <property name="upper">32</property>
@@ -1298,6 +1305,32 @@
                     <property name="top-attach">5</property>
                   </packing>
                 </child>
+                <child>
+                  <object class="GtkLabel">
+                    <property name="name">voice_display_speakers_grace_period_label</property>
+                    <property name="visible">True</property>
+                    <property name="can-focus">False</property>
+                    <property name="label" translatable="yes">Speakers Grace Period</property>
+                    <property name="xalign">0</property>
+                  </object>
+                  <packing>
+                    <property name="left-attach">4</property>
+                    <property name="top-attach">6</property>
+                  </packing>
+                </child>
+                <child>
+                  <object class="GtkSpinButton">
+                    <property name="name">voice_display_speakers_grace_period</property>
+                    <property name="visible">True</property>
+                    <property name="can-focus">True</property>
+                    <property name="adjustment">voice_display_speakers_grace_period_adj</property>
+                    <signal name="value-changed" handler="voice_display_speakers_grace_period" swapped="no"/>
+                  </object>
+                  <packing>
+                    <property name="left-attach">5</property>
+                    <property name="top-attach">6</property>
+                  </packing>
+                </child>
                 <child>
                   <object class="GtkLabel">
                     <property name="name">voice_nick_length_label</property>
@@ -1308,7 +1341,7 @@
                   </object>
                   <packing>
                     <property name="left-attach">4</property>
-                    <property name="top-attach">6</property>
+                    <property name="top-attach">7</property>
                   </packing>
                 </child>
                 <child>
@@ -1321,7 +1354,7 @@
                   </object>
                   <packing>
                     <property name="left-attach">5</property>
-                    <property name="top-attach">6</property>
+                    <property name="top-attach">7</property>
                   </packing>
                 </child>
                 <child>
diff --git a/discover_overlay/settings_window.py b/discover_overlay/settings_window.py
index 2cac43c..5a20496 100644
--- a/discover_overlay/settings_window.py
+++ b/discover_overlay/settings_window.py
@@ -310,6 +310,9 @@ class MainSettingsWindow():
         self.widget['voice_display_speakers_only'].set_active(
             config.getboolean("main", "only_speaking", fallback=False))
 
+        self.widget['voice_display_speakers_grace_period'].set_value(
+            config.getint("main", "only_speaking_grace", fallback=0))
+
         self.widget['voice_show_test_content'].set_active(
             config.getboolean("main", "show_dummy", fallback=False))
 
@@ -802,6 +805,9 @@ class MainSettingsWindow():
     def voice_display_speakers_only(self, button):
         self.config_set("main", "only_speaking", "%s" % (button.get_active()))
 
+    def voice_display_speakers_grace_period(self, button):
+        self.config_set("main", "only_speaking_grace", "%s" % (int(button.get_value())))
+
     def voice_toggle_test_content(self, button):
         self.config_set("main", "show_dummy", "%s" % (button.get_active()))
 
diff --git a/discover_overlay/voice_overlay.py b/discover_overlay/voice_overlay.py
index c345a8e..344be92 100644
--- a/discover_overlay/voice_overlay.py
+++ b/discover_overlay/voice_overlay.py
@@ -19,6 +19,7 @@ import cairo
 import sys
 import locale
 import pkg_resources
+from time import perf_counter
 from .overlay import OverlayWindow
 from .image_getter import get_surface, draw_img_to_rect, draw_img_to_mask
 # pylint: disable=wrong-import-order
@@ -44,6 +45,9 @@ class VoiceOverlayWindow(OverlayWindow):
         self.avatars = {}
         self.avatar_masks = {}
 
+        # Cache for when somebody last spoke, used for "only_speaking" grace period
+        self.speaker_cache = {}
+
         self.dummy_data = []
         mostly_false = [False, False, False, False, False, False, False, True]
         for i in range(0, 100):
@@ -303,6 +307,12 @@ class VoiceOverlayWindow(OverlayWindow):
         """
         self.only_speaking = only_speaking
 
+    def set_only_speaking_grace_period(self, grace_period):
+        """
+        Set grace period before hiding people who are not talking
+        """
+        self.only_speaking_grace_period = grace_period
+
     def set_highlight_self(self, highlight_self):
         """
         Set if the overlay should highlight the user
@@ -468,11 +478,27 @@ class VoiceOverlayWindow(OverlayWindow):
             else:
                 user["friendlyname"] = user["username"]
 
-            # Remove users that arent speaking
+            # Remove users that haven't spoken within the grace period
             if self.only_speaking:
                 speaking = "speaking" in user and user["speaking"]
+
+                # Update the speaker cache
+                if speaking:
+                    self.speaker_cache[user["username"]] = perf_counter()
+
                 if not speaking:
-                    if user in users_to_draw:
+                    grace = self.only_speaking_grace_period
+
+                    if (
+                        grace > 0
+                        and (last_spoke := self.speaker_cache.get(user["username"]))
+                        and (perf_counter() - last_spoke) < grace
+                    ):
+                        # The user spoke within the grace period, so don't hide
+                        # them just yet
+                        pass
+
+                    elif user in users_to_draw:
                         users_to_draw.remove(user)
 
         if self.highlight_self: