diff --git a/src/video_core/texture_cache/texture_cache.cpp b/src/video_core/texture_cache/texture_cache.cpp
index 89b263b78..57422196d 100644
--- a/src/video_core/texture_cache/texture_cache.cpp
+++ b/src/video_core/texture_cache/texture_cache.cpp
@@ -404,27 +404,19 @@ void TextureCache::RefreshImage(Image& image, Vulkan::Scheduler* custom_schedule
 
     const VAddr image_addr = image.info.guest_address;
     const size_t image_size = image.info.guest_size_bytes;
-    vk::Buffer buffer{};
-    u32 offset{};
-    if (auto upload_buffer = tile_manager.TryDetile(image); upload_buffer) {
-        buffer = *upload_buffer;
-    } else {
-        const auto [vk_buffer, buf_offset] = buffer_cache.ObtainTempBuffer(image_addr, image_size);
-        buffer = vk_buffer->Handle();
-        offset = buf_offset;
-
-        // The obtained buffer may be written by a shader so we need to emit a barrier to prevent
-        // RAW hazard
-        if (auto barrier = vk_buffer->GetBarrier(vk::AccessFlagBits2::eTransferRead,
-                                                 vk::PipelineStageFlagBits2::eTransfer)) {
-            auto dependencies = vk::DependencyInfo{
-                .bufferMemoryBarrierCount = 1,
-                .pBufferMemoryBarriers = &barrier.value(),
-            };
-            cmdbuf.pipelineBarrier2(dependencies);
-        }
+    const auto [vk_buffer, buf_offset] = buffer_cache.ObtainTempBuffer(image_addr, image_size);
+    // The obtained buffer may be written by a shader so we need to emit a barrier to prevent RAW
+    // hazard
+    if (auto barrier = vk_buffer->GetBarrier(vk::AccessFlagBits2::eTransferRead,
+                                             vk::PipelineStageFlagBits2::eTransfer)) {
+        auto dependencies = vk::DependencyInfo{
+            .bufferMemoryBarrierCount = 1,
+            .pBufferMemoryBarriers = &barrier.value(),
+        };
+        cmdbuf.pipelineBarrier2(dependencies);
     }
 
+    const auto [buffer, offset] = tile_manager.TryDetile(vk_buffer->Handle(), buf_offset, image);
     for (auto& copy : image_copy) {
         copy.bufferOffset += offset;
     }
diff --git a/src/video_core/texture_cache/tile_manager.cpp b/src/video_core/texture_cache/tile_manager.cpp
index 7fe5598d4..93a3bf319 100644
--- a/src/video_core/texture_cache/tile_manager.cpp
+++ b/src/video_core/texture_cache/tile_manager.cpp
@@ -377,9 +377,10 @@ void TileManager::FreeBuffer(ScratchBuffer buffer) {
     vmaDestroyBuffer(instance.GetAllocator(), buffer.first, buffer.second);
 }
 
-std::optional<vk::Buffer> TileManager::TryDetile(Image& image) {
+std::pair<vk::Buffer, u32> TileManager::TryDetile(vk::Buffer in_buffer, u32 in_offset,
+                                                  Image& image) {
     if (!image.info.props.is_tiled) {
-        return std::nullopt;
+        return {in_buffer, in_offset};
     }
 
     const auto* detiler = GetDetiler(image);
@@ -388,19 +389,10 @@ std::optional<vk::Buffer> TileManager::TryDetile(Image& image) {
             LOG_ERROR(Render_Vulkan, "Unsupported tiled image: {} ({})",
                       vk::to_string(image.info.pixel_format), NameOf(image.info.tiling_mode));
         }
-        return std::nullopt;
+        return {in_buffer, in_offset};
     }
 
-    // Prepare input buffer
     const u32 image_size = image.info.guest_size_bytes;
-    const auto [in_buffer, in_offset] = [&] -> std::pair<vk::Buffer, u32> {
-        // Request temporary host buffer for larger sizes.
-        auto in_buffer = AllocBuffer(image_size);
-        const auto addr = reinterpret_cast<const void*>(image.info.guest_address);
-        Upload(in_buffer, addr, image_size);
-        scheduler.DeferOperation([=, this]() { FreeBuffer(in_buffer); });
-        return {in_buffer.first, 0};
-    }();
 
     // Prepare output buffer
     auto out_buffer = AllocBuffer(image_size, true);
@@ -471,7 +463,7 @@ std::optional<vk::Buffer> TileManager::TryDetile(Image& image) {
                            vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
                            {}, post_barrier, {});
 
-    return {out_buffer.first};
+    return {out_buffer.first, 0};
 }
 
 } // namespace VideoCore
diff --git a/src/video_core/texture_cache/tile_manager.h b/src/video_core/texture_cache/tile_manager.h
index 0baabf98d..d0e5eb0f3 100644
--- a/src/video_core/texture_cache/tile_manager.h
+++ b/src/video_core/texture_cache/tile_manager.h
@@ -39,7 +39,7 @@ public:
     TileManager(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler);
     ~TileManager();
 
-    std::optional<vk::Buffer> TryDetile(Image& image);
+    std::pair<vk::Buffer, u32> TryDetile(vk::Buffer in_buffer, u32 in_offset, Image& image);
 
     ScratchBuffer AllocBuffer(u32 size, bool is_storage = false);
     void Upload(ScratchBuffer buffer, const void* data, size_t size);