diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index 014b4a318e..f14ddfb79b 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -68,7 +68,6 @@ void BufferCache<P>::RunGarbageCollector() {
         DeleteBuffer(buffer_id);
         return false;
     };
-    lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up);
 }
 
 template <class P>
@@ -1595,10 +1594,8 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
     const auto size = buffer.SizeBytes();
     if (insert) {
         total_used_memory += Common::AlignUp(size, 1024);
-        buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick));
     } else {
         total_used_memory -= Common::AlignUp(size, 1024);
-        lru_cache.Free(buffer.getLRUID());
     }
     const DAddr device_addr_begin = buffer.CpuAddr();
     const DAddr device_addr_end = device_addr_begin + size;
@@ -1615,9 +1612,7 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
 
 template <class P>
 void BufferCache<P>::TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept {
-    if (buffer_id != NULL_BUFFER_ID) {
-        lru_cache.Touch(buffer.getLRUID(), frame_tick);
-    }
+    if (buffer_id != NULL_BUFFER_ID) {}
 }
 
 template <class P>
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 08524bd854..b17fed1b6d 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -23,7 +23,6 @@
 #include "common/common_types.h"
 #include "common/div_ceil.h"
 #include "common/literals.h"
-#include "common/lru_cache.h"
 #include "common/range_sets.h"
 #include "common/scope_exit.h"
 #include "common/settings.h"
@@ -506,11 +505,6 @@ private:
     size_t immediate_buffer_capacity = 0;
     Common::ScratchBuffer<u8> immediate_buffer_alloc;
 
-    struct LRUItemParams {
-        using ObjectType = BufferId;
-        using TickType = u64;
-    };
-    Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
     u64 frame_tick = 0;
     u64 total_used_memory = 0;
     u64 minimum_memory = 0;
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 0587d7b724..0d96fa436a 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -102,7 +102,6 @@ struct ImageBase {
     VAddr cpu_addr_end = 0;
 
     u64 modification_tick = 0;
-    size_t lru_index = SIZE_MAX;
 
     std::array<u32, MAX_MIP_LEVELS> mip_level_offsets{};
 
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 71210ffe6e..a03181de14 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -120,13 +120,11 @@ template <class P>
 void TextureCache<P>::RunGarbageCollector() {
     bool high_priority_mode = false;
     bool aggressive_mode = false;
-    u64 ticks_to_destroy = 0;
     size_t num_iterations = 0;
 
     const auto Configure = [&](bool allow_aggressive) {
         high_priority_mode = total_used_memory >= expected_memory;
         aggressive_mode = allow_aggressive && total_used_memory >= critical_memory;
-        ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL;
         num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10);
     };
 
@@ -195,32 +193,10 @@ void TextureCache<P>::RunGarbageCollector() {
         }
         return false;
     };
-
-    // Aggressively clear massive sparse textures
-    if (total_used_memory >= expected_memory) {
-        lru_cache.ForEachItemBelow(frame_tick, [&](ImageId image_id) {
-            auto& image = slot_images[image_id];
-            // Only target sparse textures that are old enough
-            if (lowmemorydevice &&
-                image.info.is_sparse &&
-                image.guest_size_bytes >= 256_MiB &&
-                image.allocation_tick < frame_tick - 3) {
-                LOG_DEBUG(HW_GPU, "GC targeting old sparse texture at 0x{:X} ({} MiB, age: {} frames)",
-                         image.gpu_addr, image.guest_size_bytes / (1024 * 1024),
-                         frame_tick - image.allocation_tick);
-                return Cleanup(image_id);
-            }
-            return false;
-        });
-    }
-
     Configure(false);
-    lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
-
     // If pressure is still too high, prune aggressively.
     if (total_used_memory >= critical_memory) {
         Configure(true);
-        lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
     }
 }
 
@@ -2028,7 +2004,6 @@ std::pair<u32, u32> TextureCache<P>::PrepareDmaImage(ImageId dst_id, GPUVAddr ba
     const auto base = image.TryFindBase(base_addr);
     PrepareImage(dst_id, mark_as_modified, false);
     const auto& new_image = slot_images[dst_id];
-    lru_cache.Touch(new_image.lru_index, frame_tick);
     return std::make_pair(base->level, base->layer);
 }
 
@@ -2377,7 +2352,6 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
         tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
     }
     total_used_memory += Common::AlignUp(tentative_size, 1024);
-    image.lru_index = lru_cache.Insert(image_id, frame_tick);
 
     ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
         (*channel_state->gpu_page_table)[page].push_back(image_id);
@@ -2411,7 +2385,6 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
                "Trying to unregister an already registered image");
     image.flags &= ~ImageFlagBits::Registered;
     image.flags &= ~ImageFlagBits::BadOverlap;
-    lru_cache.Free(image.lru_index);
     const auto& clear_page_table =
         [image_id](u64 page, ankerl::unordered_dense::map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>& selected_page_table) {
             const auto page_it = selected_page_table.find(page);
@@ -2738,7 +2711,6 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool
     if (is_modification) {
         MarkModification(image);
     }
-    lru_cache.Touch(image.lru_index, frame_tick);
 }
 
 template <class P>
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 4b4061f21d..0c93fd8901 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -22,7 +22,6 @@
 #include "common/common_types.h"
 #include "common/hash.h"
 #include "common/literals.h"
-#include "common/lru_cache.h"
 #include <ranges>
 #include "common/scratch_buffer.h"
 #include "common/slot_vector.h"
@@ -510,12 +509,6 @@ private:
     std::deque<std::vector<AsyncBuffer>> async_buffers;
     std::deque<AsyncBuffer> async_buffers_death_ring;
 
-    struct LRUItemParams {
-        using ObjectType = ImageId;
-        using TickType = u64;
-    };
-    Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
-
  #ifdef YUZU_LEGACY
     static constexpr size_t TICKS_TO_DESTROY = 6;
  #else