Introduce new manual descriptor set reservation method that prevents it from being used by something else while owned by an array

src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs

@@ -326,6 +326,10 @@ namespace Ryujinx.Graphics.Shader.Translation
             {
                 if (arrayLength > 1 && (setIndex = _gpuAccessor.CreateExtraSet()) >= 0)
                 {
+                    // We reserved an "extra set" for the array.
+                    // In this case the binding is always the first one (0).
+                    // Using separate sets for array is better as we need to do less descriptor set updates.
+
                     binding = 0;
                 }
                 else

src/Ryujinx.Graphics.Vulkan/ImageArray.cs

@@ -2,6 +2,7 @@ using Ryujinx.Graphics.GAL;
 using Silk.NET.Vulkan;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 
 namespace Ryujinx.Graphics.Vulkan
 {
@@ -29,6 +30,10 @@ namespace Ryujinx.Graphics.Vulkan
         private int _cachedCommandBufferIndex;
         private int _cachedSubmissionCount;
 
+        private ShaderCollection _cachedDscProgram;
+        private int _cachedDscSetIndex;
+        private int _cachedDscIndex;
+
         private readonly bool _isBuffer;
 
         private int _bindCount;
@@ -206,8 +211,8 @@ namespace Ryujinx.Graphics.Vulkan
                 return _cachedDescriptorSets;
             }
 
-            program.UpdateDescriptorCacheCommandBufferIndex(cbs.CommandBufferIndex);
+            _cachedDscProgram?.ReleaseManualDescriptorSetCollection(_cachedDscSetIndex, _cachedDscIndex);
-            var dsc = program.GetNewDescriptorSetCollection(setIndex, out _).Get(cbs);
+            var dsc = program.GetNewManualDescriptorSetCollection(cbs.CommandBufferIndex, setIndex, out _cachedDscIndex).Get(cbs);
 
             DescriptorSetTemplate template = program.Templates[setIndex];
 
@@ -225,6 +230,8 @@ namespace Ryujinx.Graphics.Vulkan
             var sets = dsc.GetSets();
             templateUpdater.Commit(_gd, device, sets[0]);
             _cachedDescriptorSets = sets;
+            _cachedDscProgram = program;
+            _cachedDscSetIndex = setIndex;
 
             return sets;
         }
@@ -237,11 +244,7 @@ namespace Ryujinx.Graphics.Vulkan
         public void DecrementBindCount()
         {
             int newBindCount = --_bindCount;
-
+            Debug.Assert(newBindCount >= 0);
-            if (newBindCount < 0)
-            {
-                throw new Exception("huh?");
-            }
         }
     }
 }

src/Ryujinx.Graphics.Vulkan/PipelineLayoutCacheEntry.cs

@@ -3,6 +3,7 @@ using Silk.NET.Vulkan;
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Runtime.InteropServices;
 
 namespace Ryujinx.Graphics.Vulkan
 {
@@ -27,6 +28,24 @@ namespace Ryujinx.Graphics.Vulkan
         private int _dsLastCbIndex;
         private int _dsLastSubmissionCount;
 
+        private struct ManualDescriptorSetEntry
+        {
+            public Auto<DescriptorSetCollection> DescriptorSet;
+            public int CbIndex;
+            public int CbSubmissionCount;
+            public bool InUse;
+
+            public ManualDescriptorSetEntry(Auto<DescriptorSetCollection> descriptorSet, int cbIndex, int cbSubmissionCount, bool inUse)
+            {
+                DescriptorSet = descriptorSet;
+                CbIndex = cbIndex;
+                CbSubmissionCount = cbSubmissionCount;
+                InUse = inUse;
+            }
+        }
+
+        private readonly List<ManualDescriptorSetEntry>[] _manualDsCache;
+
         private readonly Dictionary<long, DescriptorSetTemplate> _pdTemplates;
         private readonly ResourceDescriptorCollection _pdDescriptors;
         private long _lastPdUsage;
@@ -50,6 +69,7 @@ namespace Ryujinx.Graphics.Vulkan
             }
 
             _dsCacheCursor = new int[setsCount];
+            _manualDsCache = new List<ManualDescriptorSetEntry>[setsCount];
         }
 
         public PipelineLayoutCacheEntry(
@@ -124,6 +144,51 @@ namespace Ryujinx.Graphics.Vulkan
             return list[index];
         }
 
+        public Auto<DescriptorSetCollection> GetNewManualDescriptorSetCollection(int commandBufferIndex, int setIndex, out int cacheIndex)
+        {
+            int submissionCount = _gd.CommandBufferPool.GetSubmissionCount(commandBufferIndex);
+
+            var list = _manualDsCache[setIndex] ??= new();
+            var span = CollectionsMarshal.AsSpan(list);
+
+            for (int index = 0; index < span.Length; index++)
+            {
+                ref ManualDescriptorSetEntry entry = ref span[index];
+
+                if (!entry.InUse && (entry.CbIndex != commandBufferIndex || entry.CbSubmissionCount != submissionCount))
+                {
+                    entry.InUse = true;
+                    entry.CbIndex = commandBufferIndex;
+                    entry.CbSubmissionCount = submissionCount;
+
+                    cacheIndex = index;
+
+                    return entry.DescriptorSet;
+                }
+            }
+
+            var dsc = _descriptorSetManager.AllocateDescriptorSet(
+                _gd.Api,
+                DescriptorSetLayouts[setIndex],
+                _poolSizes[setIndex],
+                setIndex,
+                _consumedDescriptorsPerSet[setIndex],
+                false);
+
+            cacheIndex = list.Count;
+            list.Add(new ManualDescriptorSetEntry(dsc, commandBufferIndex, submissionCount, inUse: true));
+
+            return dsc;
+        }
+
+        public void ReleaseManualDescriptorSetCollection(int setIndex, int cacheIndex)
+        {
+            var list = _manualDsCache[setIndex];
+            var span = CollectionsMarshal.AsSpan(list);
+
+            span[cacheIndex].InUse = false;
+        }
+
         private static Span<DescriptorPoolSize> GetDescriptorPoolSizes(Span<DescriptorPoolSize> output, ResourceDescriptorCollection setDescriptor, uint multiplier)
         {
             int count = 0;
@@ -204,6 +269,21 @@ namespace Ryujinx.Graphics.Vulkan
                     }
                 }
 
+                for (int i = 0; i < _manualDsCache.Length; i++)
+                {
+                    if (_manualDsCache[i] == null)
+                    {
+                        continue;
+                    }
+
+                    for (int j = 0; j < _manualDsCache[i].Count; j++)
+                    {
+                        _manualDsCache[i][j].DescriptorSet.Dispose();
+                    }
+
+                    _manualDsCache[i].Clear();
+                }
+
                 _gd.Api.DestroyPipelineLayout(_device, PipelineLayout, null);
 
                 for (int i = 0; i < DescriptorSetLayouts.Length; i++)

src/Ryujinx.Graphics.Vulkan/ShaderCollection.cs

@@ -604,6 +604,16 @@ namespace Ryujinx.Graphics.Vulkan
             return _plce.GetNewDescriptorSetCollection(setIndex, out isNew);
         }
 
+        public Auto<DescriptorSetCollection> GetNewManualDescriptorSetCollection(int commandBufferIndex, int setIndex, out int cacheIndex)
+        {
+            return _plce.GetNewManualDescriptorSetCollection(commandBufferIndex, setIndex, out cacheIndex);
+        }
+
+        public void ReleaseManualDescriptorSetCollection(int setIndex, int cacheIndex)
+        {
+            _plce.ReleaseManualDescriptorSetCollection(setIndex, cacheIndex);
+        }
+
         public bool HasSameLayout(ShaderCollection other)
         {
             return other != null && _plce == other._plce;

src/Ryujinx.Graphics.Vulkan/TextureArray.cs

@@ -2,6 +2,7 @@ using Ryujinx.Graphics.GAL;
 using Silk.NET.Vulkan;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 
 namespace Ryujinx.Graphics.Vulkan
 {
@@ -29,6 +30,10 @@ namespace Ryujinx.Graphics.Vulkan
         private int _cachedCommandBufferIndex;
         private int _cachedSubmissionCount;
 
+        private ShaderCollection _cachedDscProgram;
+        private int _cachedDscSetIndex;
+        private int _cachedDscIndex;
+
         private readonly bool _isBuffer;
 
         private int _bindCount;
@@ -222,8 +227,8 @@ namespace Ryujinx.Graphics.Vulkan
                 return _cachedDescriptorSets;
             }
 
-            program.UpdateDescriptorCacheCommandBufferIndex(cbs.CommandBufferIndex);
+            _cachedDscProgram?.ReleaseManualDescriptorSetCollection(_cachedDscSetIndex, _cachedDscIndex);
-            var dsc = program.GetNewDescriptorSetCollection(setIndex, out _).Get(cbs);
+            var dsc = program.GetNewManualDescriptorSetCollection(cbs.CommandBufferIndex, setIndex, out _cachedDscIndex).Get(cbs);
 
             DescriptorSetTemplate template = program.Templates[setIndex];
 
@@ -241,6 +246,8 @@ namespace Ryujinx.Graphics.Vulkan
             var sets = dsc.GetSets();
             templateUpdater.Commit(_gd, device, sets[0]);
             _cachedDescriptorSets = sets;
+            _cachedDscProgram = program;
+            _cachedDscSetIndex = setIndex;
 
             return sets;
         }
@@ -253,11 +260,7 @@ namespace Ryujinx.Graphics.Vulkan
         public void DecrementBindCount()
         {
             int newBindCount = --_bindCount;
-
+            Debug.Assert(newBindCount >= 0);
-            if (newBindCount < 0)
-            {
-                throw new Exception("huh?");
-            }
         }
     }
 }