[shader_recompiler, vulkan, qcom, tu] Provide handling to missing shader subgroups on QCOM (#4508)

This works provides better emulation on certain GS missing for shader invocation data + geometry pass fallback in qcom/turnip drivers, some of those were provided as community patch by @a.kamaltdinov; which was refined to address the requirements on forced gpu reads; which used to kill some performance on low end devices; now, the stability with some of the graphical problems have been addressed and balanced. Shuffle shaders (SHFFL) are not actually provided on QCOM and turnip drivers, so any game that previously required this stage draw, will now avoid being ignored and tax some performance based on the complexity of the stage draw from now on. This will also address partially some tessellation problems as well.

_Special Thanks:_

1.- @a.kamaltdinov (for the initial patch "early fix for nier" provided for #4325)
2.- @PavelBARABANOV (because the making the cherry-pick easier for a new PR)

Co-authored-by: PavelBARABANOV <pavelbarabanov94@gmail.com>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4508
Reviewed-by: lizzie <lizzie@eden-emu.dev>
Reviewed-by: crueter <crueter@eden-emu.dev>

src/shader_recompiler/frontend/maxwell/translate_program.cpp

@@ -5,7 +5,10 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include <algorithm>
+#include <array>
+#include <bitset>
 #include <memory>
+#include <optional>
 #include <vector>
 #include <queue>
 
@@ -172,7 +175,17 @@ std::map<IR::Attribute, IR::Attribute> GenerateLegacyToGenericMappings(
 void EmitGeometryPassthrough(IR::IREmitter& ir, const IR::Program& program,
                              const Shader::VaryingState& passthrough_mask,
                              bool passthrough_position,
-                             std::optional<IR::Attribute> passthrough_layer_attr) {
+                             std::optional<IR::Attribute> passthrough_layer_attr,
+                             std::optional<IR::Reg> viewport_mask_reg) {
+    constexpr std::array CULLED_POSITION{2.0f, 2.0f, 2.0f, 1.0f};
+    IR::U1 culled{ir.Imm1(false)};
+    IR::U32 viewport{ir.Imm32(0)};
+    if (viewport_mask_reg) {
+        const IR::U32 mask{ir.GetReg(*viewport_mask_reg)};
+        const IR::U32 lowest_bit{ir.BitwiseAnd(mask, IR::U32{ir.INeg(mask)})};
+        culled = ir.IEqual(mask, ir.Imm32(0));
+        viewport = IR::U32{ir.Select(culled, ir.Imm32(0), ir.FindUMsb(lowest_bit))};
+    }
     for (u32 i = 0; i < program.output_vertices; i++) {
         // Assign generics from input
         for (u32 j = 0; j < 32; j++) {
@@ -190,10 +203,19 @@ void EmitGeometryPassthrough(IR::IREmitter& ir, const IR::Program& program,
         if (passthrough_position) {
             // Assign position from input
             const IR::Attribute attr = IR::Attribute::PositionX;
-            ir.SetAttribute(attr + 0, ir.GetAttribute(attr + 0, ir.Imm32(i)), ir.Imm32(0));
-            ir.SetAttribute(attr + 1, ir.GetAttribute(attr + 1, ir.Imm32(i)), ir.Imm32(0));
-            ir.SetAttribute(attr + 2, ir.GetAttribute(attr + 2, ir.Imm32(i)), ir.Imm32(0));
-            ir.SetAttribute(attr + 3, ir.GetAttribute(attr + 3, ir.Imm32(i)), ir.Imm32(0));
+            for (u32 component = 0; component < 4; ++component) {
+                IR::F32 value{ir.GetAttribute(attr + component, ir.Imm32(i))};
+                if (viewport_mask_reg) {
+                    value = IR::F32{
+                        ir.Select(culled, ir.Imm32(CULLED_POSITION[component]), value)};
+                }
+                ir.SetAttribute(attr + component, value, ir.Imm32(0));
+            }
+        }
+
+        if (viewport_mask_reg) {
+            ir.SetAttribute(IR::Attribute::ViewportIndex, ir.BitCast<IR::F32>(viewport),
+                            ir.Imm32(0));
         }
 
         if (passthrough_layer_attr) {
@@ -219,19 +241,91 @@ u32 GetOutputTopologyVertices(OutputTopology output_topology) {
     }
 }
 
+std::optional<IR::Reg> FindFreeRegister(const IR::Program& program) {
+    std::bitset<IR::NUM_REGS> used;
+    for (IR::Block* const block : program.blocks) {
+        for (const IR::Inst& inst : block->Instructions()) {
+            const IR::Opcode opcode{inst.GetOpcode()};
+            if (opcode == IR::Opcode::GetRegister || opcode == IR::Opcode::SetRegister) {
+                used.set(IR::RegIndex(inst.Arg(0).Reg()));
+            }
+        }
+    }
+    for (size_t index = IR::NUM_USER_REGS; index-- > 0;) {
+        if (!used.test(index)) {
+            return static_cast<IR::Reg>(index);
+        }
+    }
+    return std::nullopt;
+}
+
+std::optional<IR::Reg> LowerViewportMask(const IR::Program& program) {
+    const std::optional<IR::Reg> reg{FindFreeRegister(program)};
+    if (!reg || program.blocks.empty()) {
+        return std::nullopt;
+    }
+    bool stores_mask{};
+    for (IR::Block* const block : program.blocks) {
+        for (IR::Inst& inst : block->Instructions()) {
+            if (inst.GetOpcode() != IR::Opcode::SetAttribute ||
+                inst.Arg(0).Attribute() != IR::Attribute::ViewportMask) {
+                continue;
+            }
+            IR::IREmitter ir{*block, IR::Block::InstructionList::s_iterator_to(inst)};
+            ir.SetReg(*reg, ir.BitCast<IR::U32>(IR::F32{inst.Arg(1)}));
+            inst.Invalidate();
+            stores_mask = true;
+        }
+    }
+    if (!stores_mask) {
+        return std::nullopt;
+    }
+    IR::Block& entry{*program.blocks.front()};
+    IR::IREmitter ir{entry, entry.begin()};
+    ir.SetReg(*reg, ir.Imm32(1));
+    return reg;
+}
+
 void LowerGeometryPassthrough(const IR::Program& program, const HostTranslateInfo& host_info) {
+    std::optional<IR::Reg> viewport_mask_reg;
+    if (!host_info.support_viewport_mask) {
+        viewport_mask_reg = LowerViewportMask(program);
+    }
     for (IR::Block* const block : program.blocks) {
         for (IR::Inst& inst : block->Instructions()) {
             if (inst.GetOpcode() == IR::Opcode::Epilogue) {
                 IR::IREmitter ir{*block, IR::Block::InstructionList::s_iterator_to(inst)};
                 EmitGeometryPassthrough(
                     ir, program, program.info.passthrough,
-                    program.info.passthrough.AnyComponent(IR::Attribute::PositionX), {});
+                    program.info.passthrough.AnyComponent(IR::Attribute::PositionX), {},
+                    viewport_mask_reg);
             }
         }
     }
 }
 
+void TightenOutputVertices(IR::Program& program) {
+    if (program.stage != Stage::Geometry || program.is_geometry_passthrough) {
+        return;
+    }
+    const bool has_loops = std::ranges::any_of(program.syntax_list, [](const auto& node) {
+        return node.type == IR::AbstractSyntaxNode::Type::Loop;
+    });
+    if (has_loops) {
+        return;
+    }
+    u32 num_emits = 0;
+    for (IR::Block* const block : program.blocks) {
+        for (const IR::Inst& inst : block->Instructions()) {
+            if (inst.GetOpcode() == IR::Opcode::EmitVertex) {
+                ++num_emits;
+            }
+        }
+    }
+    if (num_emits != 0) {
+        program.output_vertices = (std::min)(program.output_vertices, num_emits);
+    }
+}
 } // Anonymous namespace
 
 IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Block>& block_pool,
@@ -299,12 +393,14 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo
     Optimization::PositionPass(env, program);
 
     Optimization::GlobalMemoryToStorageBufferPass(program, normalized_host_info);
+    Optimization::GeometryCompactionPass(program, normalized_host_info);
     Optimization::TexturePass(env, program, normalized_host_info);
 
     if (Settings::values.resolution_info.active) {
         Optimization::RescalingPass(program);
     }
     Optimization::DeadCodeEliminationPass(program);
+    TightenOutputVertices(program);
     if (Settings::values.renderer_debug) {
         Optimization::VerificationPass(program);
     }
@@ -433,7 +529,7 @@ IR::Program GenerateGeometryPassthrough(ObjectPool<IR::Inst>& inst_pool,
 
     IR::IREmitter ir{*current_block};
     EmitGeometryPassthrough(ir, program, program.info.stores, true,
-                            source_program.info.emulated_layer);
+                            source_program.info.emulated_layer, std::nullopt);
 
     IR::Block* return_block{block_pool.Create(inst_pool)};
     IR::IREmitter{*return_block}.Epilogue();