From 92749f011ee038fc13919c83e26e9063cd4e6a6d Mon Sep 17 00:00:00 2001
From: kd-11 <karokidii@gmail.com>
Date: Sat, 21 Aug 2021 17:03:44 +0300
Subject: [PATCH] vk: Experimental workaround to just disable compression

Applies some flags to disable aggressive optimizations on some hw.
Currently applies to AMD cards which will avoid compression if mutable
format is enabled.
This is a temporary workaround.
---
 rpcs3/Emu/RSX/VK/VKRenderTargets.h | 55 ++++++++++++++++++------------
 1 file changed, 33 insertions(+), 22 deletions(-)

diff --git a/rpcs3/Emu/RSX/VK/VKRenderTargets.h b/rpcs3/Emu/RSX/VK/VKRenderTargets.h
index a6731725dc..4dad8ba826 100644
--- a/rpcs3/Emu/RSX/VK/VKRenderTargets.h
+++ b/rpcs3/Emu/RSX/VK/VKRenderTargets.h
@@ -89,33 +89,36 @@ namespace vk
 		using download_buffer_object = void*;
 		using barrier_descriptor_t = rsx::deferred_clipped_region<vk::render_target*>;
 
-		static VkFlags get_attachment_compression_usage_flags()
+		static std::pair<VkImageUsageFlags, VkImageCreateFlags> get_attachment_create_flags(u8 samples)
 		{
-			if (g_cfg.video.strict_rendering_mode)
+			if (g_cfg.video.strict_rendering_mode || samples > 1)
 			{
-				return 0;
+				return {};
 			}
 
+			// Workarounds to force transition to GENERAL to decompress.
+			// Fixes corruption in FBO loops for ANV and RADV.
 			switch (vk::get_driver_vendor())
 			{
-			case driver_vendor::NVIDIA:
-			case driver_vendor::INTEL:
-			case driver_vendor::AMD: // TODO
-				return 0;
-
-			// Workaround to force transition to GENERAL to decompress.
-			// Fixes corruption in FBO loops for ANV and RADV.
 			case driver_vendor::ANV:
-				return VK_IMAGE_USAGE_STORAGE_BIT;
+				return { VK_IMAGE_USAGE_STORAGE_BIT, 0 };
+			case driver_vendor::AMD:
 			case driver_vendor::RADV:
-				// Only needed for GFX10+
-				return (vk::get_chip_family() >= chip_class::AMD_navi1x) ?
-					VK_IMAGE_USAGE_STORAGE_BIT : 0;
-
+				if (vk::get_chip_family() >= chip_class::AMD_navi1x)
+				{
+					// Only needed for GFX10+
+					return { 0, VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT };
+				}
+				break;
 			default:
 				rsx_log.error("Unknown driver vendor!");
-				return 0;
+				[[ fallthrough ]];
+			case driver_vendor::NVIDIA:
+			case driver_vendor::INTEL:
+				break;
 			}
+
+			return {};
 		}
 
 		static std::unique_ptr<vk::render_target> create_new_surface(
@@ -141,10 +144,12 @@ namespace vk
 				sample_layout = rsx::surface_sample_layout::null;
 			}
 
-			VkImageUsageFlags usage_flags = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+			auto [usage_flags, create_flags] = get_attachment_create_flags(samples);
+			usage_flags |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
+
 			if (samples == 1) [[likely]]
 			{
-				usage_flags |= get_attachment_compression_usage_flags() | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
+				usage_flags |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
 			}
 			else
 			{
@@ -163,7 +168,9 @@ namespace vk
 				VK_IMAGE_LAYOUT_UNDEFINED,
 				VK_IMAGE_TILING_OPTIMAL,
 				usage_flags,
-				0, VMM_ALLOCATION_POOL_SURFACE_CACHE, RSX_FORMAT_CLASS_COLOR);
+				create_flags,
+				VMM_ALLOCATION_POOL_SURFACE_CACHE,
+				RSX_FORMAT_CLASS_COLOR);
 
 			rtt->set_debug_name(fmt::format("RTV @0x%x", address));
 			rtt->change_layout(cmd, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
@@ -192,7 +199,6 @@ namespace vk
 			vk::render_device& device, vk::command_buffer& cmd)
 		{
 			const VkFormat requested_format = vk::get_compatible_depth_surface_format(device.get_formats_support(), format);
-			VkImageUsageFlags usage_flags = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
 
 			u8 samples;
 			rsx::surface_sample_layout sample_layout;
@@ -207,9 +213,12 @@ namespace vk
 				sample_layout = rsx::surface_sample_layout::null;
 			}
 
+			auto [usage_flags, create_flags] = get_attachment_create_flags(samples);
+			usage_flags |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
+
 			if (samples == 1) [[likely]]
 			{
-				usage_flags |= get_attachment_compression_usage_flags() | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+				usage_flags |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
 			}
 
 			std::unique_ptr<vk::render_target> ds;
@@ -224,7 +233,9 @@ namespace vk
 				VK_IMAGE_LAYOUT_UNDEFINED,
 				VK_IMAGE_TILING_OPTIMAL,
 				usage_flags,
-				0, VMM_ALLOCATION_POOL_SURFACE_CACHE, rsx::classify_format(format));
+				create_flags,
+				VMM_ALLOCATION_POOL_SURFACE_CACHE,
+				rsx::classify_format(format));
 
 			ds->set_debug_name(fmt::format("DSV @0x%x", address));
 			ds->change_layout(cmd, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL);