Vulkan: Feedback loop detection and barriers (#7226)

* Vulkan: Feedback loop improvements

This PR allows the Vulkan backend to detect attachment feedback loops. These are currently used in the following ways:

- Partial use of VK_EXT_attachment_feedback_loop_layout
  - All renderable textures have AttachmentFeedbackLoopBitExt
  - Compile pipelines with Color/DepthStencil feedback loop flags when present
- Support using FragmentBarrier for feedback loops (fixes regressions from https://github.com/Ryujinx/Ryujinx/pull/7012 )

TODO:
- AMD GPUs may need layout transitions for it to properly allow textures to be used in feedback loops.
- Use dynamic state for feedback loops. The background pipeline will always miss since feedback loop state isn't known on the GPU project.
- How is the barrier dependency flag used? (DXVK just ignores it, there's no vulkan validation...)
- Improve subpass dependencies to fix validation errors

* Mark field readonly

* Add feedback loop dynamic state

* fix: add MoltenVK resolver workaround

fix: add MoltenVK resolver workaround

* Formatting

* Fix more complaints

* RADV dcc workaround

* Use dynamic state properly, cleanup.

* Use aspects flags in more places

src/Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -2,6 +2,7 @@ using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Shader;
 using Silk.NET.Vulkan;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Numerics;
 using System.Runtime.CompilerServices;
@@ -33,6 +34,7 @@ namespace Ryujinx.Graphics.Vulkan
         public readonly Action EndRenderPassDelegate;
 
         protected PipelineDynamicState DynamicState;
+        protected bool IsMainPipeline;
         private PipelineState _newState;
         private bool _graphicsStateDirty;
         private bool _computeStateDirty;
@@ -85,6 +87,9 @@ namespace Ryujinx.Graphics.Vulkan
         private bool _tfEnabled;
         private bool _tfActive;
 
+        private FeedbackLoopAspects _feedbackLoop;
+        private bool _passWritesDepthStencil;
+
         private readonly PipelineColorBlendAttachmentState[] _storedBlend;
         public ulong DrawCount { get; private set; }
         public bool RenderPassActive { get; private set; }
@@ -126,7 +131,7 @@ namespace Ryujinx.Graphics.Vulkan
 
         public void Initialize()
         {
-            _descriptorSetUpdater.Initialize();
+            _descriptorSetUpdater.Initialize(IsMainPipeline);
 
             QuadsToTrisPattern = new IndexBufferPattern(Gd, 4, 6, 0, new[] { 0, 1, 2, 0, 2, 3 }, 4, false);
             TriFanToTrisPattern = new IndexBufferPattern(Gd, 3, 3, 2, new[] { int.MinValue, -1, 0 }, 1, true);
@@ -814,6 +819,8 @@ namespace Ryujinx.Graphics.Vulkan
             _newState.DepthTestEnable = depthTest.TestEnable;
             _newState.DepthWriteEnable = depthTest.WriteEnable;
             _newState.DepthCompareOp = depthTest.Func.Convert();
+
+            UpdatePassDepthStencil();
             SignalStateChange();
         }
 
@@ -1079,6 +1086,8 @@ namespace Ryujinx.Graphics.Vulkan
             _newState.StencilFrontPassOp = stencilTest.FrontDpPass.Convert();
             _newState.StencilFrontDepthFailOp = stencilTest.FrontDpFail.Convert();
             _newState.StencilFrontCompareOp = stencilTest.FrontFunc.Convert();
+
+            UpdatePassDepthStencil();
             SignalStateChange();
         }
 
@@ -1426,7 +1435,23 @@ namespace Ryujinx.Graphics.Vulkan
                 }
             }
 
+            if (IsMainPipeline)
+            {
+                FramebufferParams?.ClearBindings();
+            }
+
             FramebufferParams = new FramebufferParams(Device, colors, depthStencil);
+
+            if (IsMainPipeline)
+            {
+                FramebufferParams.AddBindings();
+
+                _newState.FeedbackLoopAspects = FeedbackLoopAspects.None;
+                _bindingBarriersDirty = true;
+            }
+
+            _passWritesDepthStencil = false;
+            UpdatePassDepthStencil();
             UpdatePipelineAttachmentFormats();
         }
 
@@ -1493,11 +1518,82 @@ namespace Ryujinx.Graphics.Vulkan
                 }
             }
 
-            Gd.Barriers.Flush(Cbs, _program, RenderPassActive, _rpHolder, EndRenderPassDelegate);
+            Gd.Barriers.Flush(Cbs, _program, _feedbackLoop != 0, RenderPassActive, _rpHolder, EndRenderPassDelegate);
 
             _descriptorSetUpdater.UpdateAndBindDescriptorSets(Cbs, PipelineBindPoint.Compute);
         }
 
+        private bool ChangeFeedbackLoop(FeedbackLoopAspects aspects)
+        {
+            if (_feedbackLoop != aspects)
+            {
+                if (Gd.Capabilities.SupportsDynamicAttachmentFeedbackLoop)
+                {
+                    DynamicState.SetFeedbackLoop(aspects);
+                }
+                else
+                {
+                    _newState.FeedbackLoopAspects = aspects;
+                }
+
+                _feedbackLoop = aspects;
+
+                return true;
+            }
+
+            return false;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool UpdateFeedbackLoop()
+        {
+            List<TextureView> hazards = _descriptorSetUpdater.FeedbackLoopHazards;
+
+            if ((hazards?.Count ?? 0) > 0)
+            {
+                FeedbackLoopAspects aspects = 0;
+
+                foreach (TextureView view in hazards)
+                {
+                    // May need to enforce feedback loop layout here in the future.
+                    // Though technically, it should always work with the general layout.
+
+                    if (view.Info.Format.IsDepthOrStencil())
+                    {
+                        if (_passWritesDepthStencil)
+                        {
+                            // If depth/stencil isn't written in the pass, it doesn't count as a feedback loop.
+
+                            aspects |= FeedbackLoopAspects.Depth;
+                        }
+                    }
+                    else
+                    {
+                        aspects |= FeedbackLoopAspects.Color;
+                    }
+                }
+
+                return ChangeFeedbackLoop(aspects);
+            }
+            else if (_feedbackLoop != 0)
+            {
+                return ChangeFeedbackLoop(FeedbackLoopAspects.None);
+            }
+
+            return false;
+        }
+
+        private void UpdatePassDepthStencil()
+        {
+            if (!RenderPassActive)
+            {
+                _passWritesDepthStencil = false;
+            }
+
+            // Stencil test being enabled doesn't necessarily mean a write, but it's not critical to check.
+            _passWritesDepthStencil |= (_newState.DepthTestEnable && _newState.DepthWriteEnable) || _newState.StencilTestEnable;
+        }
+
         private bool RecreateGraphicsPipelineIfNeeded()
         {
             if (AutoFlush.ShouldFlushDraw(DrawCount))
@@ -1505,7 +1601,7 @@ namespace Ryujinx.Graphics.Vulkan
                 Gd.FlushAllCommands();
             }
 
-            DynamicState.ReplayIfDirty(Gd.Api, CommandBuffer);
+            DynamicState.ReplayIfDirty(Gd, CommandBuffer);
 
             if (_needsIndexBufferRebind && _indexBufferPattern == null)
             {
@@ -1539,7 +1635,15 @@ namespace Ryujinx.Graphics.Vulkan
                 _vertexBufferUpdater.Commit(Cbs);
             }
 
-            if (_graphicsStateDirty || Pbp != PipelineBindPoint.Graphics)
+            if (_bindingBarriersDirty)
+            {
+                // Stale barriers may have been activated by switching program. Emit any that are relevant.
+                _descriptorSetUpdater.InsertBindingBarriers(Cbs);
+
+                _bindingBarriersDirty = false;
+            }
+
+            if (UpdateFeedbackLoop() || _graphicsStateDirty || Pbp != PipelineBindPoint.Graphics)
             {
                 if (!CreatePipeline(PipelineBindPoint.Graphics))
                 {
@@ -1548,17 +1652,9 @@ namespace Ryujinx.Graphics.Vulkan
 
                 _graphicsStateDirty = false;
                 Pbp = PipelineBindPoint.Graphics;
-
-                if (_bindingBarriersDirty)
-                {
-                    // Stale barriers may have been activated by switching program. Emit any that are relevant.
-                    _descriptorSetUpdater.InsertBindingBarriers(Cbs);
-
-                    _bindingBarriersDirty = false;
-                }
             }
 
-            Gd.Barriers.Flush(Cbs, _program, RenderPassActive, _rpHolder, EndRenderPassDelegate);
+            Gd.Barriers.Flush(Cbs, _program, _feedbackLoop != 0, RenderPassActive, _rpHolder, EndRenderPassDelegate);
 
             _descriptorSetUpdater.UpdateAndBindDescriptorSets(Cbs, PipelineBindPoint.Graphics);