Implement virtual buffer dependencies (#6190)

* Implement virtual buffer copies

* Introduce TranslateAndCreateMultiBuffersPhysicalOnly, use it for copy and clear

* Rename VirtualBufferCache to VirtualRangeCache

* Fix potential issue where virtual range could exist in the cache, without a physical buffer

* Fix bug that could cause copy with negative size on CopyToDependantVirtualBuffer

* Remove virtual copy back for SyncAction

* GetData XML docs

* Make field readonly

* Fix virtual buffer modification tracking

* Remove CopyFromDependantVirtualBuffers from ExternalFlush

* Move things around a little to avoid perf impact

- Inline null check for CopyFromDependantVirtualBuffers
- Remove extra method call for SynchronizeMemoryWithVirtualCopyBack, prefer calling CopyFromDependantVirtualBuffers separately

* Fix up XML doc

---------

Co-authored-by: riperiperi <rhy3756547@hotmail.com>

src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs

@@ -3,6 +3,7 @@ using Ryujinx.Memory.Range;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 
 namespace Ryujinx.Graphics.Gpu.Memory
 {
@@ -46,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache;
         private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache;
         private bool _pruneCaches;
+        private int _virtualModifiedSequenceNumber;
 
         public event Action NotifyBuffersModified;
 
@@ -125,7 +127,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
         /// <summary>
         /// Performs address translation of the GPU virtual address, and creates
-        /// new buffers, if needed, for the specified range.
+        /// new physical and virtual buffers, if needed, for the specified range.
         /// </summary>
         /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
         /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
@@ -138,12 +140,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 return new MultiRange(MemoryManager.PteUnmapped, size);
             }
 
-            bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
-
             // Fast path not taken for non-contiguous ranges,
             // since multi-range buffers are not coalesced, so a buffer that covers
             // the entire cached range might not actually exist.
-            if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
+            if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
                 range.Count == 1)
             {
                 return range;
@@ -154,6 +154,50 @@ namespace Ryujinx.Graphics.Gpu.Memory
             return range;
         }
 
+        /// <summary>
+        /// Performs address translation of the GPU virtual address, and creates
+        /// new physical buffers, if needed, for the specified range.
+        /// </summary>
+        /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
+        /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
+        /// <param name="size">Size in bytes of the buffer</param>
+        /// <returns>Physical ranges of the buffer, after address translation</returns>
+        public MultiRange TranslateAndCreateMultiBuffersPhysicalOnly(MemoryManager memoryManager, ulong gpuVa, ulong size)
+        {
+            if (gpuVa == 0)
+            {
+                return new MultiRange(MemoryManager.PteUnmapped, size);
+            }
+
+            // Fast path not taken for non-contiguous ranges,
+            // since multi-range buffers are not coalesced, so a buffer that covers
+            // the entire cached range might not actually exist.
+            if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
+                range.Count == 1)
+            {
+                return range;
+            }
+
+            for (int i = 0; i < range.Count; i++)
+            {
+                MemoryRange subRange = range.GetSubRange(i);
+
+                if (subRange.Address != MemoryManager.PteUnmapped)
+                {
+                    if (range.Count > 1)
+                    {
+                        CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
+                    }
+                    else
+                    {
+                        CreateBuffer(subRange.Address, subRange.Size);
+                    }
+                }
+            }
+
+            return range;
+        }
+
         /// <summary>
         /// Creates a new buffer for the specified range, if it does not yet exist.
         /// This can be used to ensure the existance of a buffer.
@@ -263,41 +307,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 }
             }
 
-            BufferRange[] storages = new BufferRange[range.Count];
+            MultiRangeBuffer multiRangeBuffer;
+
             MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
 
             ulong alignmentMask = SparseBufferAlignmentSize - 1;
 
-            for (int i = 0; i < range.Count; i++)
+            if (_context.Capabilities.SupportsSparseBuffer)
             {
-                MemoryRange subRange = range.GetSubRange(i);
+                BufferRange[] storages = new BufferRange[range.Count];
+
+                for (int i = 0; i < range.Count; i++)
+                {
+                    MemoryRange subRange = range.GetSubRange(i);
+
+                    if (subRange.Address != MemoryManager.PteUnmapped)
+                    {
+                        ulong endAddress = subRange.Address + subRange.Size;
+
+                        ulong alignedAddress = subRange.Address & ~alignmentMask;
+                        ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
+                        ulong alignedSize = alignedEndAddress - alignedAddress;
+
+                        Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
+                        BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
+
+                        alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+                        storages[i] = bufferRange;
+                    }
+                    else
+                    {
+                        ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
+
+                        alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
+                        storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
+                    }
+                }
+
+                multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
+            }
+            else
+            {
+                for (int i = 0; i < range.Count; i++)
+                {
+                    MemoryRange subRange = range.GetSubRange(i);
+
+                    if (subRange.Address != MemoryManager.PteUnmapped)
+                    {
+                        ulong endAddress = subRange.Address + subRange.Size;
+
+                        ulong alignedAddress = subRange.Address & ~alignmentMask;
+                        ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
+                        ulong alignedSize = alignedEndAddress - alignedAddress;
+
+                        alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+                    }
+                    else
+                    {
+                        ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
+
+                        alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
+                    }
+                }
+
+                multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges));
+
+                UpdateVirtualBufferDependencies(multiRangeBuffer);
+            }
+
+            _multiRangeBuffers.Add(multiRangeBuffer);
+        }
+
+        /// <summary>
+        /// Adds two-way dependencies to all physical buffers contained within a given virtual buffer.
+        /// </summary>
+        /// <param name="virtualBuffer">Virtual buffer to have dependencies added</param>
+        private void UpdateVirtualBufferDependencies(MultiRangeBuffer virtualBuffer)
+        {
+            virtualBuffer.ClearPhysicalDependencies();
+
+            ulong dstOffset = 0;
+
+            HashSet<Buffer> physicalBuffers = new();
+
+            for (int i = 0; i < virtualBuffer.Range.Count; i++)
+            {
+                MemoryRange subRange = virtualBuffer.Range.GetSubRange(i);
 
                 if (subRange.Address != MemoryManager.PteUnmapped)
                 {
-                    ulong endAddress = subRange.Address + subRange.Size;
+                    Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
 
-                    ulong alignedAddress = subRange.Address & ~alignmentMask;
-                    ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
-                    ulong alignedSize = alignedEndAddress - alignedAddress;
-
-                    Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
-                    BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
-
-                    storages[i] = bufferRange;
-                    alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+                    virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size);
+                    physicalBuffers.Add(buffer);
                 }
-                else
-                {
-                    ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
 
-                    storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
-                    alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
-                }
+                dstOffset += subRange.Size;
             }
 
-            MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
-
-            _multiRangeBuffers.Add(multiRangeBuffer);
+            foreach (var buffer in physicalBuffers)
+            {
+                buffer.CopyToDependantVirtualBuffer(virtualBuffer);
+            }
         }
 
         /// <summary>
@@ -620,8 +731,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="size">Size in bytes of the copy</param>
         public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
         {
-            MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
-            MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
+            MultiRange srcRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, srcVa, size);
+            MultiRange dstRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, dstVa, size);
 
             if (srcRange.Count == 1 && dstRange.Count == 1)
             {
@@ -701,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 dstBuffer.ClearModified(dstAddress, size);
                 memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer);
             }
+
+            dstBuffer.CopyToDependantVirtualBuffers(dstAddress, size);
         }
 
         /// <summary>
@@ -715,7 +828,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="value">Value to be written into the buffer</param>
         public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
         {
-            MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
+            MultiRange range = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, gpuVa, size);
 
             for (int index = 0; index < range.Count; index++)
             {
@@ -727,6 +840,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
 
                 memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
+
+                buffer.CopyToDependantVirtualBuffers(subRange.Address, subRange.Size);
             }
         }
 
@@ -806,6 +921,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 }
             }
 
+            if (write && buffer != null && !_context.Capabilities.SupportsSparseBuffer)
+            {
+                buffer.AddModifiedRegion(range, ++_virtualModifiedSequenceNumber);
+            }
+
             return buffer;
         }
 
@@ -825,6 +945,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
             {
                 buffer = _buffers.FindFirstOverlap(address, size);
 
+                buffer.CopyFromDependantVirtualBuffers();
                 buffer.SynchronizeMemory(address, size);
 
                 if (write)
@@ -849,14 +970,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
             if (range.Count == 1)
             {
                 MemoryRange subRange = range.GetSubRange(0);
-                SynchronizeBufferRange(subRange.Address, subRange.Size);
+                SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: true);
             }
             else
             {
                 for (int index = 0; index < range.Count; index++)
                 {
                     MemoryRange subRange = range.GetSubRange(index);
-                    SynchronizeBufferRange(subRange.Address, subRange.Size);
+                    SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: false);
                 }
             }
         }
@@ -866,12 +987,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </summary>
         /// <param name="address">Start address of the memory range</param>
         /// <param name="size">Size in bytes of the memory range</param>
-        private void SynchronizeBufferRange(ulong address, ulong size)
+        /// <param name="copyBackVirtual">Whether virtual buffers that uses this buffer as backing memory should have its data copied back if modified</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void SynchronizeBufferRange(ulong address, ulong size, bool copyBackVirtual)
         {
             if (size != 0)
             {
                 Buffer buffer = _buffers.FindFirstOverlap(address, size);
 
+                if (copyBackVirtual)
+                {
+                    buffer.CopyFromDependantVirtualBuffers();
+                }
+
                 buffer.SynchronizeMemory(address, size);
             }
         }