diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index c460d90d23..b3984d042d 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -121,6 +121,15 @@ void BufferCache<P>::WriteMemory(DAddr device_addr, u64 size) {
     memory_tracker.MarkRegionAsCpuModified(device_addr, size);
 }
 
+template <class P>
+void BufferCache<P>::UnmapMemory(DAddr device_addr, u64 size) {
+    if (memory_tracker.IsRegionGpuModified(device_addr, size)) {
+        ClearDownload(device_addr, size);
+        gpu_modified_ranges.Subtract(device_addr, size);
+    }
+    memory_tracker.UnmarkRegionAsCpuModified(device_addr, size);
+}
+
 template <class P>
 void BufferCache<P>::CachedWriteMemory(DAddr device_addr, u64 size) {
     const bool is_dirty = IsRegionRegistered(device_addr, size);
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 8c300905fc..bea4068e6d 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -219,6 +219,8 @@ public:
 
     void WriteMemory(DAddr device_addr, u64 size);
 
+    void UnmapMemory(DAddr device_addr, u64 size);
+
     void CachedWriteMemory(DAddr device_addr, u64 size);
 
     bool OnCPUWrite(DAddr device_addr, u64 size);
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 3d24904a53..25d23b057c 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -595,7 +595,7 @@ void RasterizerOpenGL::UnmapMemory(DAddr addr, u64 size) {
     }
     {
         std::scoped_lock lock{buffer_cache.mutex};
-        buffer_cache.WriteMemory(addr, size);
+        buffer_cache.UnmapMemory(addr, size);
     }
     shader_cache.OnCacheInvalidation(addr, size);
 }
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index ed3f8c08a9..62f7fec8c2 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -758,7 +758,7 @@ void RasterizerVulkan::UnmapMemory(DAddr addr, u64 size) {
     }
     {
         std::scoped_lock lock{buffer_cache.mutex};
-        buffer_cache.WriteMemory(addr, size);
+        buffer_cache.UnmapMemory(addr, size);
     }
     pipeline_cache.OnCacheInvalidation(addr, size);
 }