diff --git a/Source/Core/DolphinQt/GameList/GameTracker.cpp b/Source/Core/DolphinQt/GameList/GameTracker.cpp
index 8f6f76e39a..5b15bfc53c 100644
--- a/Source/Core/DolphinQt/GameList/GameTracker.cpp
+++ b/Source/Core/DolphinQt/GameList/GameTracker.cpp
@@ -338,7 +338,8 @@ QSet<QString> GameTracker::FindMissingFiles(const QString& dir)
   while (it->hasNext())
   {
     QString path = QFileInfo(it->next()).canonicalFilePath();
-    m_tracked_files.remove(path);
+    if (m_tracked_files.contains(path))
+      missing_files.remove(path);
   }
 
   return missing_files;