don't bind unused color buffers

src/shader_recompiler/frontend/translate/export.cpp

@@ -71,6 +71,9 @@ void Translator::EmitExport(const GcnInst& inst) {
             ir.SetAttribute(attrib, comp, swizzle(i));
         }
     }
+    if (IR::IsMrt(attrib)) {
+        info.mrt_mask |= 1u << u8(attrib);
+    }
 }
 
 } // namespace Shader::Gcn

src/shader_recompiler/info.h

@@ -172,6 +172,7 @@ struct Info {
     bool uses_fp64{};
     bool uses_step_rates{};
     bool translation_failed{}; // indicates that shader has unsupported instructions
+    u8 mrt_mask{0u};
 
     explicit Info(Stage stage_, ShaderParams params)
         : stage{stage_}, pgm_hash{params.hash}, pgm_base{params.Base()},

src/video_core/renderer_vulkan/vk_graphics_pipeline.h

@@ -33,6 +33,7 @@ struct GraphicsPipelineKey {
     Liverpool::DepthControl depth_stencil;
     u32 depth_bias_enable;
     u32 num_samples;
+    u32 mrt_mask;
     Liverpool::StencilControl stencil;
     Liverpool::PrimitiveType prim_type;
     u32 enable_primitive_restart;
@@ -74,6 +75,10 @@ public:
         return key.write_masks;
     }
 
+    auto GetMrtMask() const {
+        return key.mrt_mask;
+    }
+
     bool IsDepthEnabled() const {
         return key.depth_stencil.depth_enable.Value();
     }

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -234,39 +234,6 @@ bool PipelineCache::RefreshGraphicsKey() {
     key.front_face = regs.polygon_control.front_face;
     key.num_samples = regs.aa_config.NumSamples();
 
-    const auto skip_cb_binding =
-        regs.color_control.mode == AmdGpu::Liverpool::ColorControl::OperationMode::Disable;
-
-    // `RenderingInfo` is assumed to be initialized with a contiguous array of valid color
-    // attachments. This might be not a case as HW color buffers can be bound in an arbitrary order.
-    // We need to do some arrays compaction at this stage
-    key.color_formats.fill(vk::Format::eUndefined);
-    key.blend_controls.fill({});
-    key.write_masks.fill({});
-    key.mrt_swizzles.fill(Liverpool::ColorBuffer::SwapMode::Standard);
-    int remapped_cb{};
-    for (auto cb = 0u; cb < Liverpool::NumColorBuffers; ++cb) {
-        auto const& col_buf = regs.color_buffers[cb];
-        if (skip_cb_binding || !col_buf || !regs.color_target_mask.GetMask(cb)) {
-            continue;
-        }
-        const auto base_format =
-            LiverpoolToVK::SurfaceFormat(col_buf.info.format, col_buf.NumFormat());
-        const bool is_vo_surface = renderer->IsVideoOutSurface(col_buf);
-        key.color_formats[remapped_cb] = LiverpoolToVK::AdjustColorBufferFormat(
-            base_format, col_buf.info.comp_swap.Value(), false /*is_vo_surface*/);
-        if (base_format == key.color_formats[remapped_cb]) {
-            key.mrt_swizzles[remapped_cb] = col_buf.info.comp_swap.Value();
-        }
-        key.blend_controls[remapped_cb] = regs.blend_control[cb];
-        key.blend_controls[remapped_cb].enable.Assign(key.blend_controls[remapped_cb].enable &&
-                                                      !col_buf.info.blend_bypass);
-        key.write_masks[remapped_cb] = vk::ColorComponentFlags{regs.color_target_mask.GetMask(cb)};
-        key.cb_shader_mask.SetMask(remapped_cb, regs.color_shader_mask.GetMask(cb));
-
-        ++remapped_cb;
-    }
-
     u32 binding{};
     for (u32 i = 0; i < MaxShaderStages; i++) {
         if (!regs.stage_enable.IsStageEnabled(i)) {
@@ -309,6 +276,46 @@ bool PipelineCache::RefreshGraphicsKey() {
 
         std::tie(infos[i], modules[i], key.stage_hashes[i]) = GetProgram(stage, params, binding);
     }
+
+    const auto* fs_info = infos[u32(Shader::Stage::Fragment)];
+    key.mrt_mask = fs_info ? fs_info->mrt_mask : 0u;
+
+    const auto skip_cb_binding =
+        regs.color_control.mode == AmdGpu::Liverpool::ColorControl::OperationMode::Disable;
+
+    // `RenderingInfo` is assumed to be initialized with a contiguous array of valid color
+    // attachments. This might be not a case as HW color buffers can be bound in an arbitrary order.
+    // We need to do some arrays compaction at this stage
+    key.color_formats.fill(vk::Format::eUndefined);
+    key.blend_controls.fill({});
+    key.write_masks.fill({});
+    key.mrt_swizzles.fill(Liverpool::ColorBuffer::SwapMode::Standard);
+    int remapped_cb{};
+    for (auto cb = 0u; cb < Liverpool::NumColorBuffers; ++cb) {
+        auto const& col_buf = regs.color_buffers[cb];
+        if (skip_cb_binding || !col_buf || !regs.color_target_mask.GetMask(cb)) {
+            continue;
+        }
+        if ((key.mrt_mask & (1u << cb)) == 0) {
+            continue;
+        }
+        const auto base_format =
+            LiverpoolToVK::SurfaceFormat(col_buf.info.format, col_buf.NumFormat());
+        const bool is_vo_surface = renderer->IsVideoOutSurface(col_buf);
+        key.color_formats[remapped_cb] = LiverpoolToVK::AdjustColorBufferFormat(
+            base_format, col_buf.info.comp_swap.Value(), false /*is_vo_surface*/);
+        if (base_format == key.color_formats[remapped_cb]) {
+            key.mrt_swizzles[remapped_cb] = col_buf.info.comp_swap.Value();
+        }
+        key.blend_controls[remapped_cb] = regs.blend_control[cb];
+        key.blend_controls[remapped_cb].enable.Assign(key.blend_controls[remapped_cb].enable &&
+                                                      !col_buf.info.blend_bypass);
+        key.write_masks[remapped_cb] = vk::ColorComponentFlags{regs.color_target_mask.GetMask(cb)};
+        key.cb_shader_mask.SetMask(remapped_cb, regs.color_shader_mask.GetMask(cb));
+
+        ++remapped_cb;
+    }
+
     return true;
 }
 

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -62,7 +62,7 @@ void Rasterizer::Draw(bool is_indexed, u32 index_offset) {
     buffer_cache.BindVertexBuffers(vs_info);
     const u32 num_indices = buffer_cache.BindIndexBuffer(is_indexed, index_offset);
 
-    BeginRendering();
+    BeginRendering(*pipeline);
     UpdateDynamicState(*pipeline);
 
     const auto [vertex_offset, instance_offset] = vs_info.GetDrawOffsets();
@@ -102,7 +102,7 @@ void Rasterizer::DrawIndirect(bool is_indexed, VAddr address, u32 offset, u32 si
     buffer_cache.BindVertexBuffers(vs_info);
     const u32 num_indices = buffer_cache.BindIndexBuffer(is_indexed, 0);
 
-    BeginRendering();
+    BeginRendering(*pipeline);
     UpdateDynamicState(*pipeline);
 
     const auto [buffer, base] = buffer_cache.ObtainBuffer(address, size, true);
@@ -179,7 +179,7 @@ void Rasterizer::Finish() {
     scheduler.Finish();
 }
 
-void Rasterizer::BeginRendering() {
+void Rasterizer::BeginRendering(const GraphicsPipeline& pipeline) {
     const auto& regs = liverpool->regs;
     RenderState state;
 
@@ -199,6 +199,13 @@ void Rasterizer::BeginRendering() {
             continue;
         }
 
+        // Skip stale color buffers if shader doesn't output to them. Otherwise it will perform
+        // an unnecessary transition and may result in state conflict if the resource is already
+        // bound for reading.
+        if ((pipeline.GetMrtMask() & (1 << col_buf_id)) == 0) {
+            continue;
+        }
+
         const auto& hint = liverpool->last_cb_extent[col_buf_id];
         VideoCore::ImageInfo image_info{col_buf, hint};
         VideoCore::ImageViewInfo view_info{col_buf, false /*!!image.info.usage.vo_buffer*/};

src/video_core/renderer_vulkan/vk_rasterizer.h

@@ -52,7 +52,7 @@ public:
     void Finish();
 
 private:
-    void BeginRendering();
+    void BeginRendering(const GraphicsPipeline& pipeline);
 
     void UpdateDynamicState(const GraphicsPipeline& pipeline);
     void UpdateViewportScissorState();