diff --git a/discover_overlay/discord_connector.py b/discover_overlay/discord_connector.py
index 3d59df3..89418d2 100644
--- a/discover_overlay/discord_connector.py
+++ b/discover_overlay/discord_connector.py
@@ -307,6 +307,7 @@ class DiscordConnector:
                 # We've joined a room... but where?
                 if j["data"]["user"]["id"] == self.user["id"]:
                     self.find_user()
+                self.userlist[thisuser["id"]]["lastspoken"] = time.perf_counter()
             elif j["evt"] == "VOICE_STATE_DELETE":
                 self.list_altered = True
                 self.set_in_room(j["data"]["user"]["id"], False)
@@ -319,7 +320,7 @@ class DiscordConnector:
             elif j["evt"] == "SPEAKING_START":
                 self.list_altered = True
                 self.userlist[j["data"]["user_id"]]["speaking"] = True
-                self.userlist[j["data"]["user_id"]]["lastspoken"] = time.time()
+                self.userlist[j["data"]["user_id"]]["lastspoken"] = time.perf_counter()
                 self.set_in_room(j["data"]["user_id"], True)
             elif j["evt"] == "SPEAKING_STOP":
                 self.list_altered = True
diff --git a/discover_overlay/voice_overlay.py b/discover_overlay/voice_overlay.py
index 75745b7..23fe463 100644
--- a/discover_overlay/voice_overlay.py
+++ b/discover_overlay/voice_overlay.py
@@ -44,9 +44,6 @@ class VoiceOverlayWindow(OverlayWindow):
         self.avatars = {}
         self.avatar_masks = {}
 
-        # Cache for when somebody last spoke, used for "only_speaking" grace period
-        self.speaker_cache = {}
-
         self.dummy_data = []
         mostly_false = [False, False, False, False, False, False, False, True]
         for i in range(0, 100):
@@ -529,6 +526,8 @@ class VoiceOverlayWindow(OverlayWindow):
         if self.use_dummy:  # Sorting every frame is an awful idea. Maybe put this off elsewhere?
             users_to_draw = self.sort_list(self.dummy_data[0:self.dummy_count])
             userlist = self.dummy_data
+        now = perf_counter()
+
         for user in userlist:
             # Bad object equality here, so we need to reassign
             if "id" in self_user and user["id"] == self_user["id"]:
@@ -544,17 +543,16 @@ class VoiceOverlayWindow(OverlayWindow):
             if self.only_speaking:
                 speaking = "speaking" in user and user["speaking"]
 
-                # Update the speaker cache
+                # Extend timer if mid-speaking
                 if speaking:
-                    self.speaker_cache[user["username"]] = perf_counter()
-
+                    user['lastspoken'] = perf_counter()
                 if not speaking:
                     grace = self.only_speaking_grace_period
 
                     if (
                         grace > 0
-                        and (last_spoke := self.speaker_cache.get(user["username"]))
-                        and (perf_counter() - last_spoke) < grace
+                        and (last_spoke := user['lastspoken'])
+                        and (now - last_spoke) < grace
                     ):
                         # The user spoke within the grace period, so don't hide
                         # them just yet