diff --git a/src/common/lru_cache.h b/src/common/lru_cache.h
deleted file mode 100644
index 36cea5d27e..0000000000
--- a/src/common/lru_cache.h
+++ /dev/null
@@ -1,139 +0,0 @@
-// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#pragma once
-
-#include <deque>
-#include <memory>
-#include <type_traits>
-
-#include "common/common_types.h"
-
-namespace Common {
-
-template <class Traits>
-class LeastRecentlyUsedCache {
-    using ObjectType = typename Traits::ObjectType;
-    using TickType = typename Traits::TickType;
-
-    struct Item {
-        ObjectType obj;
-        TickType tick;
-        Item* next{};
-        Item* prev{};
-    };
-
-public:
-    LeastRecentlyUsedCache() : first_item{}, last_item{} {}
-    ~LeastRecentlyUsedCache() = default;
-
-    size_t Insert(ObjectType obj, TickType tick) {
-        const auto new_id = Build();
-        auto& item = item_pool[new_id];
-        item.obj = obj;
-        item.tick = tick;
-        Attach(item);
-        return new_id;
-    }
-
-    void Touch(size_t id, TickType tick) {
-        auto& item = item_pool[id];
-        if (item.tick >= tick) {
-            return;
-        }
-        item.tick = tick;
-        if (&item == last_item) {
-            return;
-        }
-        Detach(item);
-        Attach(item);
-    }
-
-    void Free(size_t id) {
-        auto& item = item_pool[id];
-        Detach(item);
-        item.prev = nullptr;
-        item.next = nullptr;
-        free_items.push_back(id);
-    }
-
-    template <typename Func>
-    void ForEachItemBelow(TickType tick, Func&& func) {
-        static constexpr bool RETURNS_BOOL =
-            std::is_same_v<std::invoke_result<Func, ObjectType>, bool>;
-        Item* iterator = first_item;
-        while (iterator) {
-            if (static_cast<s64>(tick) - static_cast<s64>(iterator->tick) < 0) {
-                return;
-            }
-            Item* next = iterator->next;
-            if constexpr (RETURNS_BOOL) {
-                if (func(iterator->obj)) {
-                    return;
-                }
-            } else {
-                func(iterator->obj);
-            }
-            iterator = next;
-        }
-    }
-
-private:
-    size_t Build() {
-        if (free_items.empty()) {
-            const size_t item_id = item_pool.size();
-            auto& item = item_pool.emplace_back();
-            item.next = nullptr;
-            item.prev = nullptr;
-            return item_id;
-        }
-        const size_t item_id = free_items.front();
-        free_items.pop_front();
-        auto& item = item_pool[item_id];
-        item.next = nullptr;
-        item.prev = nullptr;
-        return item_id;
-    }
-
-    void Attach(Item& item) {
-        if (!first_item) {
-            first_item = &item;
-        }
-        if (!last_item) {
-            last_item = &item;
-        } else {
-            item.prev = last_item;
-            last_item->next = &item;
-            item.next = nullptr;
-            last_item = &item;
-        }
-    }
-
-    void Detach(Item& item) {
-        if (item.prev) {
-            item.prev->next = item.next;
-        }
-        if (item.next) {
-            item.next->prev = item.prev;
-        }
-        if (&item == first_item) {
-            first_item = item.next;
-            if (first_item) {
-                first_item->prev = nullptr;
-            }
-        }
-        if (&item == last_item) {
-            last_item = item.prev;
-            if (last_item) {
-                last_item->next = nullptr;
-            }
-        }
-    }
-
-    std::deque<Item> item_pool;
-    std::deque<size_t> free_items;
-    Item* first_item{};
-    Item* last_item{};
-};
-
-} // namespace Common
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 0587d7b724..e121361f9b 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -102,7 +102,7 @@ struct ImageBase {
     VAddr cpu_addr_end = 0;
 
     u64 modification_tick = 0;
-    size_t lru_index = SIZE_MAX;
+    u64 last_use_tick = 0;
 
     std::array<u32, MAX_MIP_LEVELS> mip_level_offsets{};
 
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 71210ffe6e..0784bf6339 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -196,11 +196,21 @@ void TextureCache<P>::RunGarbageCollector() {
         return false;
     };
 
+    const auto CollectBelow = [this](u64 threshold) {
+        boost::container::small_vector<ImageId, 64> expired;
+        for (auto [id, image] : slot_images) {
+            if (image->last_use_tick < threshold) {
+                expired.push_back(id);
+            }
+        }
+        return expired;
+    };
+
     // Aggressively clear massive sparse textures
     if (total_used_memory >= expected_memory) {
-        lru_cache.ForEachItemBelow(frame_tick, [&](ImageId image_id) {
+        auto candidates = CollectBelow(frame_tick);
+        for (const auto image_id : candidates) {
             auto& image = slot_images[image_id];
-            // Only target sparse textures that are old enough
             if (lowmemorydevice &&
                 image.info.is_sparse &&
                 image.guest_size_bytes >= 256_MiB &&
@@ -208,19 +218,32 @@ void TextureCache<P>::RunGarbageCollector() {
                 LOG_DEBUG(HW_GPU, "GC targeting old sparse texture at 0x{:X} ({} MiB, age: {} frames)",
                          image.gpu_addr, image.guest_size_bytes / (1024 * 1024),
                          frame_tick - image.allocation_tick);
-                return Cleanup(image_id);
+                if (Cleanup(image_id)) {
+                    break;
+                }
             }
-            return false;
-        });
+        }
     }
 
     Configure(false);
-    lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
+    {
+        auto expired = CollectBelow(frame_tick - ticks_to_destroy);
+        for (const auto image_id : expired) {
+            if (Cleanup(image_id)) {
+                break;
+            }
+        }
+    }
 
     // If pressure is still too high, prune aggressively.
     if (total_used_memory >= critical_memory) {
         Configure(true);
-        lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
+        auto expired = CollectBelow(frame_tick - ticks_to_destroy);
+        for (const auto image_id : expired) {
+            if (Cleanup(image_id)) {
+                break;
+            }
+        }
     }
 }
 
@@ -2028,7 +2051,7 @@ std::pair<u32, u32> TextureCache<P>::PrepareDmaImage(ImageId dst_id, GPUVAddr ba
     const auto base = image.TryFindBase(base_addr);
     PrepareImage(dst_id, mark_as_modified, false);
     const auto& new_image = slot_images[dst_id];
-    lru_cache.Touch(new_image.lru_index, frame_tick);
+    new_image.last_use_tick = frame_tick;
     return std::make_pair(base->level, base->layer);
 }
 
@@ -2377,7 +2400,7 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
         tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
     }
     total_used_memory += Common::AlignUp(tentative_size, 1024);
-    image.lru_index = lru_cache.Insert(image_id, frame_tick);
+    image.last_use_tick = frame_tick;
 
     ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
         (*channel_state->gpu_page_table)[page].push_back(image_id);
@@ -2411,7 +2434,7 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
                "Trying to unregister an already registered image");
     image.flags &= ~ImageFlagBits::Registered;
     image.flags &= ~ImageFlagBits::BadOverlap;
-    lru_cache.Free(image.lru_index);
+
     const auto& clear_page_table =
         [image_id](u64 page, ankerl::unordered_dense::map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>& selected_page_table) {
             const auto page_it = selected_page_table.find(page);
@@ -2738,7 +2761,7 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool
     if (is_modification) {
         MarkModification(image);
     }
-    lru_cache.Touch(image.lru_index, frame_tick);
+    image.last_use_tick = frame_tick;
 }
 
 template <class P>
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 4b4061f21d..ba2af1bf44 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -22,7 +22,7 @@
 #include "common/common_types.h"
 #include "common/hash.h"
 #include "common/literals.h"
-#include "common/lru_cache.h"
+
 #include <ranges>
 #include "common/scratch_buffer.h"
 #include "common/slot_vector.h"
@@ -510,11 +510,7 @@ private:
     std::deque<std::vector<AsyncBuffer>> async_buffers;
     std::deque<AsyncBuffer> async_buffers_death_ring;
 
-    struct LRUItemParams {
-        using ObjectType = ImageId;
-        using TickType = u64;
-    };
-    Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
+
 
  #ifdef YUZU_LEGACY
     static constexpr size_t TICKS_TO_DESTROY = 6;