diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h
index bec2dac246..f080f4503c 100644
--- a/src/video_core/buffer_cache/buffer_base.h
+++ b/src/video_core/buffer_cache/buffer_base.h
@@ -109,12 +109,12 @@ public:
         return static_cast<u32>(other_cpu_addr - cpu_addr);
     }
 
-    size_t getLRUID() const noexcept {
-        return lru_id;
+    u64 GetFrameTick() const noexcept {
+        return frame_tick;
     }
 
-    void setLRUID(size_t lru_id_) {
-        lru_id = lru_id_;
+    void SetFrameTick(u64 tick) noexcept {
+        frame_tick = tick;
     }
 
     size_t SizeBytes() const {
@@ -125,7 +125,7 @@ private:
     VAddr cpu_addr = 0;
     BufferFlagBits flags{};
     int stream_score = 0;
-    size_t lru_id = SIZE_MAX;
+    u64 frame_tick = 0;
     size_t size_bytes = 0;
 };
 
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index 014b4a318e..672ecb80eb 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -58,17 +58,22 @@ void BufferCache<P>::RunGarbageCollector() {
     const bool aggressive_gc = total_used_memory >= critical_memory;
     const u64 ticks_to_destroy = aggressive_gc ? 60 : 120;
     int num_iterations = aggressive_gc ? 64 : 32;
-    const auto clean_up = [this, &num_iterations](BufferId buffer_id) {
+    const u64 threshold = frame_tick - ticks_to_destroy;
+    boost::container::small_vector<BufferId, 64> expired;
+    for (auto [id, buffer] : slot_buffers) {
+        if (buffer.GetFrameTick() < threshold) {
+            expired.push_back(id);
+        }
+    }
+    for (const auto buffer_id : expired) {
         if (num_iterations == 0) {
-            return true;
+            break;
         }
         --num_iterations;
         auto& buffer = slot_buffers[buffer_id];
         DownloadBufferMemory(buffer);
         DeleteBuffer(buffer_id);
-        return false;
-    };
-    lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up);
+    }
 }
 
 template <class P>
@@ -1595,10 +1600,9 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
     const auto size = buffer.SizeBytes();
     if (insert) {
         total_used_memory += Common::AlignUp(size, 1024);
-        buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick));
+        buffer.SetFrameTick(frame_tick);
     } else {
         total_used_memory -= Common::AlignUp(size, 1024);
-        lru_cache.Free(buffer.getLRUID());
     }
     const DAddr device_addr_begin = buffer.CpuAddr();
     const DAddr device_addr_end = device_addr_begin + size;
@@ -1616,7 +1620,7 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
 template <class P>
 void BufferCache<P>::TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept {
     if (buffer_id != NULL_BUFFER_ID) {
-        lru_cache.Touch(buffer.getLRUID(), frame_tick);
+        buffer.SetFrameTick(frame_tick);
     }
 }
 
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 08524bd854..b17fed1b6d 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -23,7 +23,6 @@
 #include "common/common_types.h"
 #include "common/div_ceil.h"
 #include "common/literals.h"
-#include "common/lru_cache.h"
 #include "common/range_sets.h"
 #include "common/scope_exit.h"
 #include "common/settings.h"
@@ -506,11 +505,6 @@ private:
     size_t immediate_buffer_capacity = 0;
     Common::ScratchBuffer<u8> immediate_buffer_alloc;
 
-    struct LRUItemParams {
-        using ObjectType = BufferId;
-        using TickType = u64;
-    };
-    Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
     u64 frame_tick = 0;
     u64 total_used_memory = 0;
     u64 minimum_memory = 0;