diff --git a/Ryujinx.Common/ReferenceEqualityComparer.cs b/Ryujinx.Common/ReferenceEqualityComparer.cs
new file mode 100644
index 0000000000..d8ec29d83a
--- /dev/null
+++ b/Ryujinx.Common/ReferenceEqualityComparer.cs
@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Ryujinx.Common
+{
+    public class ReferenceEqualityComparer<T> : IEqualityComparer<T>
+        where T : class
+    {
+        public bool Equals(T x, T y)
+        {
+            return x == y;
+        }
+
+        public int GetHashCode([DisallowNull] T obj)
+        {
+            return obj.GetHashCode();
+        }
+    }
+}
diff --git a/Ryujinx.Graphics.Gpu/Image/TextureManager.cs b/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
index 13de1b1006..87fb4161e7 100644
--- a/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
+++ b/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
@@ -61,7 +61,7 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             _cache = new AutoDeleteCache();
 
-            _modified = new HashSet<Texture>();
+            _modified = new HashSet<Texture>(new ReferenceEqualityComparer<Texture>());
         }
 
         /// <summary>