From be8f4897a2b1799e851b139bf0ad3fc1292ad56a Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Sun, 1 Jun 2025 07:30:35 +0200 Subject: [PATCH 1/7] initial commit --- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 8 +- .../Memory/BufferBackingState.cs | 7 +- .../Memory/BufferCache.cs | 107 ++- .../Memory/BufferManager.cs | 4 +- .../Memory/BufferModifiedRangeList.cs | 463 ++++++++----- .../Memory/MemoryManager.cs | 7 +- .../Memory/VirtualRangeCache.cs | 38 +- .../Range/NonOverlappingRangeList.cs | 15 +- src/Ryujinx.Memory/Range/RangeList.cs | 639 +++++++++++++++--- src/Ryujinx.Memory/Tracking/MemoryTracking.cs | 54 +- 10 files changed, 935 insertions(+), 407 deletions(-) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index a2448d76f..25034b95a 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong size, BufferStage stage, bool sparseCompatible, - IEnumerable baseBuffers = null) + IEnumerable> baseBuffers = null) { _context = context; _physicalMemory = physicalMemory; @@ -132,13 +132,13 @@ namespace Ryujinx.Graphics.Gpu.Memory { baseHandles = baseBuffers.SelectMany(buffer => { - if (buffer._useGranular) + if (buffer.Value._useGranular) { - return buffer._memoryTrackingGranular.GetHandles(); + return buffer.Value._memoryTrackingGranular.GetHandles(); } else { - return Enumerable.Repeat(buffer._memoryTracking, 1); + return Enumerable.Repeat(buffer.Value._memoryTracking, 1); } }); } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs index a9b1f50e2..61f0f9d6d 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs @@ -1,4 +1,5 @@ using Ryujinx.Graphics.GAL; +using Ryujinx.Memory.Range; using System; using System.Collections.Generic; @@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Parent buffer /// Initial buffer stage /// Buffers to inherit state from - public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable baseBuffers = null) + public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable> baseBuffers = null) { _size = (int)parent.Size; _systemMemoryType = context.Capabilities.MemoryType; @@ -102,9 +103,9 @@ namespace Ryujinx.Graphics.Gpu.Memory if (baseBuffers != null) { - foreach (Buffer buffer in baseBuffers) + foreach (RangeItem buffer in baseBuffers) { - CombineState(buffer.BackingState); + CombineState(buffer.Value.BackingState); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index d02efcb29..41d2af024 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -2,7 +2,6 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Memory.Range; using System; using System.Collections.Generic; -using System.Linq; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory @@ -42,7 +41,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly RangeList _buffers; private readonly MultiRangeList _multiRangeBuffers; - private Buffer[] _bufferOverlaps; + private RangeItem[] _bufferOverlaps; private readonly Dictionary _dirtyCache; private readonly Dictionary _modifiedCache; @@ -64,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _buffers = []; _multiRangeBuffers = []; - _bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity]; + _bufferOverlaps = new RangeItem[OverlapsBufferInitialCapacity]; _dirtyCache = new Dictionary(); @@ -79,7 +78,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Event arguments public void MemoryUnmappedHandler(object sender, UnmapEventArgs e) { - Buffer[] overlaps = new Buffer[10]; + RangeItem[] overlaps = new RangeItem[10]; int overlapCount; MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size); @@ -90,12 +89,12 @@ namespace Ryujinx.Graphics.Gpu.Memory lock (_buffers) { - overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps); + overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps).Count; } for (int i = 0; i < overlapCount; i++) { - overlaps[i].Unmapped(subRange.Address, subRange.Size); + overlaps[i].Value.Unmapped(subRange.Address, subRange.Size); } } } @@ -495,10 +494,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) { - Buffer[] overlaps = _bufferOverlaps; - int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + _buffers.Lock.EnterWriteLock(); + RangeItem[] overlaps = _bufferOverlaps; + OverlapResult result = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); - if (overlapsCount != 0) + if (result.Count != 0) { // The buffer already exists. We can just return the existing buffer // if the buffer we need is fully contained inside the overlapping buffer. @@ -507,7 +507,7 @@ namespace Ryujinx.Graphics.Gpu.Memory // old buffer(s) to the new buffer. ulong endAddress = address + size; - Buffer overlap0 = overlaps[0]; + RangeItem overlap0 = overlaps[0]; if (overlap0.Address > address || overlap0.EndAddress < endAddress) { @@ -522,39 +522,36 @@ namespace Ryujinx.Graphics.Gpu.Memory // sequential memory. // Allowing for 2 pages (rather than just one) is necessary to catch cases where the // range crosses a page, and after alignment, ends having a size of 2 pages. - if (overlapsCount == 1 && + if (result.Count == 1 && address >= overlap0.Address && endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2) { // Try to grow the buffer by 1.5x of its current size. // This improves performance in the cases where the buffer is resized often by small amounts. - ulong existingSize = overlap0.Size; + ulong existingSize = overlap0.Value.Size; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; size = Math.Max(size, growthSize); endAddress = address + size; - overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + result = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); } + + _buffers.RemoveRange(result); + + address = Math.Min(address, overlaps[0].Address); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); - for (int index = 0; index < overlapsCount; index++) + for (int index = 0; index < result.Count; index++) { - Buffer buffer = overlaps[index]; + RangeItem buffer = overlaps[index]; - anySparseCompatible |= buffer.SparseCompatible; - - address = Math.Min(address, buffer.Address); - endAddress = Math.Max(endAddress, buffer.EndAddress); - - lock (_buffers) - { - _buffers.Remove(buffer); - } + anySparseCompatible |= buffer.Value.SparseCompatible; } ulong newSize = endAddress - address; - CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, overlapsCount); + CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, result.Count); } } else @@ -562,11 +559,10 @@ namespace Ryujinx.Graphics.Gpu.Memory // No overlap, just create a new buffer. Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false); - lock (_buffers) - { - _buffers.Add(buffer); - } + _buffers.Add(buffer); } + + _buffers.Lock.ExitWriteLock(); ShrinkOverlapsBufferIfNeeded(); } @@ -582,22 +578,23 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Alignment of the start address of the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment) { - Buffer[] overlaps = _bufferOverlaps; - int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + _buffers.Lock.EnterWriteLock(); + RangeItem[] overlaps = _bufferOverlaps; + OverlapResult result = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); bool sparseAligned = alignment >= SparseBufferAlignmentSize; - if (overlapsCount != 0) + if (result.Count != 0) { // If the buffer already exists, make sure if covers the entire range, // and make sure it is properly aligned, otherwise sparse mapping may fail. ulong endAddress = address + size; - Buffer overlap0 = overlaps[0]; + RangeItem overlap0 = overlaps[0]; if (overlap0.Address > address || overlap0.EndAddress < endAddress || (overlap0.Address & (alignment - 1)) != 0 || - (!overlap0.SparseCompatible && sparseAligned)) + (!overlap0.Value.SparseCompatible && sparseAligned)) { // We need to make sure the new buffer is properly aligned. // However, after the range is aligned, it is possible that it @@ -605,35 +602,24 @@ namespace Ryujinx.Graphics.Gpu.Memory // and ensure we cover all overlaps. int oldOverlapsCount; + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); do { - for (int index = 0; index < overlapsCount; index++) - { - Buffer buffer = overlaps[index]; - - address = Math.Min(address, buffer.Address); - endAddress = Math.Max(endAddress, buffer.EndAddress); - } + address = Math.Min(address, overlaps[0].Address); address &= ~(alignment - 1); - oldOverlapsCount = overlapsCount; - overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps); + oldOverlapsCount = result.Count; + result = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps); } - while (oldOverlapsCount != overlapsCount); + while (oldOverlapsCount != result.Count); - lock (_buffers) - { - for (int index = 0; index < overlapsCount; index++) - { - _buffers.Remove(overlaps[index]); - } - } + _buffers.RemoveRange(result); ulong newSize = endAddress - address; - CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, overlapsCount); + CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, result.Count); } } else @@ -641,11 +627,9 @@ namespace Ryujinx.Graphics.Gpu.Memory // No overlap, just create a new buffer. Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned); - lock (_buffers) - { - _buffers.Add(buffer); - } + _buffers.Add(buffer); } + _buffers.Lock.ExitWriteLock(); ShrinkOverlapsBufferIfNeeded(); } @@ -661,18 +645,15 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers overlapping the range /// Total of overlaps - private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, Buffer[] overlaps, int overlapsCount) + private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, RangeItem[] overlaps, int overlapsCount) { - Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps.Take(overlapsCount)); + Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); - lock (_buffers) - { - _buffers.Add(newBuffer); - } + _buffers.Add(newBuffer); for (int index = 0; index < overlapsCount; index++) { - Buffer buffer = overlaps[index]; + Buffer buffer = overlaps[index].Value; int dstOffset = (int)(buffer.Address - newBuffer.Address); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index cb99b455b..351a00e9f 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -761,7 +761,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); + bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; BufferRange range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, bufferStage | BufferStageUtils.FromUsage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, bufferStage); @@ -798,7 +798,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); + bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; BufferRange range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, BufferStageUtils.ComputeStorage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, BufferStage.Compute); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index bccbdfd31..1e712b542 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -14,12 +14,12 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address of the range in guest memory. /// - public ulong Address { get; } + public ulong Address { get; internal set; } /// /// Size of the range in bytes. /// - public ulong Size { get; } + public ulong Size { get; internal set; } /// /// End address of the range in guest memory. @@ -77,8 +77,6 @@ namespace Ryujinx.Graphics.Gpu.Memory private BufferMigration _source; private BufferModifiedRangeList _migrationTarget; - private readonly Lock _lock = new(); - /// /// Whether the modified range list has any entries or not. /// @@ -86,10 +84,10 @@ namespace Ryujinx.Graphics.Gpu.Memory { get { - lock (_lock) - { - return Count > 0; - } + Lock.EnterReadLock(); + bool result = Count > 0; + Lock.ExitReadLock(); + return result; } } @@ -114,33 +112,39 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Action to perform for each remaining sub-range of the input range public void ExcludeModifiedRegions(ulong address, ulong size, Action action) { - lock (_lock) + // Slices a given region using the modified regions in the list. Calls the action for the new slices. + bool lockOwner = Lock.IsReadLockHeld; + if (!lockOwner) { - // Slices a given region using the modified regions in the list. Calls the action for the new slices. - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + Lock.EnterReadLock(); + } - int count = FindOverlapsNonOverlapping(address, size, ref overlaps); + FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); - for (int i = 0; i < count; i++) + for (int i = 0; i < overlaps.Length; i++) + { + BufferModifiedRange overlap = overlaps[i].Value; + + if (overlap.Address > address) { - BufferModifiedRange overlap = overlaps[i]; - - if (overlap.Address > address) - { - // The start of the remaining region is uncovered by this overlap. Call the action for it. - action(address, overlap.Address - address); - } - - // Remaining region is after this overlap. - size -= overlap.EndAddress - address; - address = overlap.EndAddress; + // The start of the remaining region is uncovered by this overlap. Call the action for it. + action(address, overlap.Address - address); } - if ((long)size > 0) - { - // If there is any region left after removing the overlaps, signal it. - action(address, size); - } + // Remaining region is after this overlap. + size -= overlap.EndAddress - address; + address = overlap.EndAddress; + } + + if (!lockOwner) + { + Lock.ExitReadLock(); + } + + if ((long)size > 0) + { + // If there is any region left after removing the overlaps, signal it. + action(address, size); } } @@ -152,51 +156,102 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size of the modified region in bytes public void SignalModified(ulong address, ulong size) { - // Must lock, as this can affect flushes from the background thread. - lock (_lock) + // We may overlap with some existing modified regions. They must be cut into by the new entry. + Lock.EnterWriteLock(); + OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size, + out ReadOnlySpan> overlaps); + + ulong endAddress = address + size; + ulong syncNumber = _context.SyncNumber; + + if (overlaps.Length == 0) { - // We may overlap with some existing modified regions. They must be cut into by the new entry. - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); + return; + } - int count = FindOverlapsNonOverlapping(address, size, ref overlaps); + BufferModifiedRange buffPost = null; + bool extendsPost = false; + bool extendsPre = false; - ulong endAddress = address + size; - ulong syncNumber = _context.SyncNumber; - - for (int i = 0; i < count; i++) + if (overlaps.Length == 1) + { + if (overlaps[0].Address == address && overlaps[0].EndAddress == endAddress) { - // The overlaps must be removed or split. + overlaps[0].Value.SyncNumber = syncNumber; + overlaps[0].Value.Parent = this; + Lock.ExitWriteLock(); + return; + } - BufferModifiedRange overlap = overlaps[i]; + if (overlaps[0].Address < address) + { + overlaps[0].Value.Size = address - overlaps[0].Address; - if (overlap.Address == address && overlap.Size == size) + extendsPre = true; + + if (overlaps[0].EndAddress > endAddress) { - // Region already exists. Just update the existing sync number. - overlap.SyncNumber = syncNumber; - overlap.Parent = this; - - return; + buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPost = true; } - - Remove(overlap); - - if (overlap.Address < address && overlap.EndAddress > address) + } + else + { + if (overlaps[0].EndAddress > endAddress) { - // A split item must be created behind this overlap. - - Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent)); + overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress; + overlaps[0].Value.Address = endAddress; } - - if (overlap.Address < endAddress && overlap.EndAddress > endAddress) + else { - // A split item must be created after this overlap. - - Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent)); + RemoveAt(result.StartIndex); } } + if (extendsPre && extendsPost) + { + Add(buffPost); + } + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); + + return; } + + BufferModifiedRange buffPre = null; + + if (overlaps[0].Address < address) + { + buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPre = true; + } + + if (overlaps[^1].EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress, + overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent); + extendsPost = true; + } + + RemoveRange(result); + + if (extendsPre) + { + Add(buffPre); + } + + if (extendsPost) + { + Add(buffPost); + } + + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); } /// @@ -208,25 +263,20 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction) { - int count = 0; + Lock.EnterReadLock(); + FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); - - // Range list must be consistent for this operation. - lock (_lock) + for (int i = 0; i < overlaps.Length; i++) { - count = FindOverlapsNonOverlapping(address, size, ref overlaps); - } - - for (int i = 0; i < count; i++) - { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; if (overlap.SyncNumber == syncNumber) { rangeAction(overlap.Address, overlap.Size); } } + + Lock.ExitReadLock(); } /// @@ -237,19 +287,15 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRanges(ulong address, ulong size, Action rangeAction) { - int count = 0; + RangeItem[] overlaps = new RangeItem[1]; + + Lock.EnterReadLock(); + OverlapResult result = FindOverlapsNonOverlapping(address, size, ref overlaps); + Lock.ExitReadLock(); - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); - - // Range list must be consistent for this operation. - lock (_lock) + for (int i = 0; i < result.Count; i++) { - count = FindOverlapsNonOverlapping(address, size, ref overlaps); - } - - for (int i = 0; i < count; i++) - { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; rangeAction(overlap.Address, overlap.Size); } } @@ -262,11 +308,10 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if a range exists in the specified region, false otherwise public bool HasRange(ulong address, ulong size) { - // Range list must be consistent for this operation. - lock (_lock) - { - return FindOverlapsNonOverlapping(address, size, ref ThreadStaticArray.Get()) > 0; - } + Lock.EnterReadLock(); + bool result = FindOverlapsNonOverlapping(address, size).Count > 0; + Lock.ExitReadLock(); + return result; } /// @@ -298,38 +343,37 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The start address of the flush range /// The end address of the flush range private void RemoveRangesAndFlush( - BufferModifiedRange[] overlaps, + RangeItem[] overlaps, int rangeCount, long highestDiff, ulong currentSync, ulong address, ulong endAddress) { - lock (_lock) + if (_migrationTarget == null) { - if (_migrationTarget == null) + ulong waitSync = currentSync + (ulong)highestDiff; + + for (int i = 0; i < rangeCount; i++) { - ulong waitSync = currentSync + (ulong)highestDiff; + BufferModifiedRange overlap = overlaps[i].Value; - for (int i = 0; i < rangeCount; i++) + long diff = (long)(overlap.SyncNumber - currentSync); + + if (diff <= highestDiff) { - BufferModifiedRange overlap = overlaps[i]; + ulong clampAddress = Math.Max(address, overlap.Address); + ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); - long diff = (long)(overlap.SyncNumber - currentSync); + Lock.EnterWriteLock(); + ClearPart(overlap, clampAddress, clampEnd); + Lock.ExitWriteLock(); - if (diff <= highestDiff) - { - ulong clampAddress = Math.Max(address, overlap.Address); - ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); - - ClearPart(overlap, clampAddress, clampEnd); - - RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); - } + RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); } - - return; } + + return; } // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine. @@ -353,22 +397,21 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong endAddress = address + size; ulong currentSync = _context.SyncNumber; - int rangeCount = 0; + int rangeCount; - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + ref RangeItem[] overlaps = ref ThreadStaticArray>.Get(); // Range list must be consistent for this operation - lock (_lock) + Lock.EnterReadLock(); + if (_migrationTarget != null) { - if (_migrationTarget != null) - { - rangeCount = -1; - } - else - { - rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps); - } + rangeCount = -1; } + else + { + rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps).Count; + } + Lock.ExitReadLock(); if (rangeCount == -1) { @@ -376,7 +419,8 @@ namespace Ryujinx.Graphics.Gpu.Memory return; } - else if (rangeCount == 0) + + if (rangeCount == 0) { return; } @@ -388,7 +432,7 @@ namespace Ryujinx.Graphics.Gpu.Memory for (int i = 0; i < rangeCount; i++) { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; long diff = (long)(overlap.SyncNumber - currentSync); @@ -419,42 +463,39 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void InheritRanges(BufferModifiedRangeList ranges, Action registerRangeAction) { - BufferModifiedRange[] inheritRanges; + ranges.Lock.EnterReadLock(); + BufferModifiedRange[] inheritRanges = ranges.ToArray(); + ranges.Lock.ExitReadLock(); - lock (ranges._lock) + // Copy over the migration from the previous range list + + BufferMigration oldMigration = ranges._source; + + BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); + ranges._parent.IncrementReferenceCount(); + + if (_source == null) { - inheritRanges = ranges.ToArray(); + // Create a new migration. + _source = new BufferMigration([span], this, _context.SyncNumber); - lock (_lock) - { - // Copy over the migration from the previous range list - - BufferMigration oldMigration = ranges._source; - - BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); - ranges._parent.IncrementReferenceCount(); - - if (_source == null) - { - // Create a new migration. - _source = new BufferMigration([span], this, _context.SyncNumber); - - _context.RegisterBufferMigration(_source); - } - else - { - // Extend the migration - _source.AddSpanToEnd(span); - } - - ranges._migrationTarget = this; - - foreach (BufferModifiedRange range in inheritRanges) - { - Add(range); - } - } + _context.RegisterBufferMigration(_source); } + else + { + // Extend the migration + _source.AddSpanToEnd(span); + } + + ranges._migrationTarget = this; + + Lock.EnterWriteLock(); + foreach (BufferModifiedRange range in inheritRanges) + { + Add(range); + } + + Lock.ExitWriteLock(); ulong currentSync = _context.SyncNumber; foreach (BufferModifiedRange range in inheritRanges) @@ -473,18 +514,18 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public void SelfMigration() { - lock (_lock) - { - BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), _parent.GetSnapshotFlushAction(), _source); - BufferMigration migration = new([span], this, _context.SyncNumber); + BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), + _parent.GetSnapshotFlushAction(), _source); + BufferMigration migration = new([span], this, _context.SyncNumber); - // Migration target is used to redirect flush actions to the latest range list, - // so we don't need to set it here. (this range list is still the latest) + // Migration target is used to redirect flush actions to the latest range list, + // so we don't need to set it here. (this range list is still the latest) - _context.RegisterBufferMigration(migration); + _context.RegisterBufferMigration(migration); - _source = migration; - } + Lock.EnterWriteLock(); + _source = migration; + Lock.ExitWriteLock(); } /// @@ -493,13 +534,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The migration to remove public void RemoveMigration(BufferMigration migration) { - lock (_lock) + Lock.EnterWriteLock(); + if (_source == migration) { - if (_source == migration) - { - _source = null; - } + _source = null; } + + Lock.ExitWriteLock(); } private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress) @@ -526,22 +567,85 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size to clear public void Clear(ulong address, ulong size) { - lock (_lock) + ulong endAddress = address + size; + Lock.EnterWriteLock(); + OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); + + if (overlaps.Length == 0) { - // This function can be called from any thread, so it cannot use the arrays for background or foreground. - BufferModifiedRange[] toClear = new BufferModifiedRange[1]; - - int rangeCount = FindOverlapsNonOverlapping(address, size, ref toClear); - - ulong endAddress = address + size; - - for (int i = 0; i < rangeCount; i++) - { - BufferModifiedRange overlap = toClear[i]; - - ClearPart(overlap, address, endAddress); - } + Lock.ExitWriteLock(); + return; } + + BufferModifiedRange buffPost = null; + bool extendsPost = false; + bool extendsPre = false; + + if (overlaps.Length == 1) + { + if (overlaps[0].Address < address) + { + overlaps[0].Value.Size = address - overlaps[0].Address; + extendsPre = true; + + if (overlaps[0].EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPost = true; + } + } + else + { + if (overlaps[^1].EndAddress > endAddress) + { + overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress; + overlaps[0].Value.Address = endAddress; + } + else + { + RemoveAt(result.StartIndex); + } + } + + if (extendsPre && extendsPost) + { + Add(buffPost); + } + + Lock.ExitWriteLock(); + return; + } + + BufferModifiedRange buffPre = null; + + if (overlaps[0].Address < address) + { + buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPre = true; + } + + if (overlaps[^1].EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress, + overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent); + extendsPost = true; + } + + RemoveRange(result); + + if (extendsPre) + { + Add(buffPre); + } + + if (extendsPost) + { + Add(buffPost); + } + + Lock.ExitWriteLock(); } /// @@ -549,10 +653,9 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public void Clear() { - lock (_lock) - { - Count = 0; - } + Lock.EnterWriteLock(); + Count = 0; + Lock.ExitWriteLock(); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index 6efb7f334..95e43e341 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -690,11 +690,8 @@ namespace Ryujinx.Graphics.Gpu.Memory if (_pageTable[l0] == null) { _pageTable[l0] = new ulong[PtLvl1Size]; - - for (ulong index = 0; index < PtLvl1Size; index++) - { - _pageTable[l0][index] = PteUnmapped; - } + + Array.Fill(_pageTable[l0], PteUnmapped); } _pageTable[l0][l1] = pte; diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index ac25b3e5d..38b628a98 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -63,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } private readonly RangeList _virtualRanges; - private VirtualRange[] _virtualRangeOverlaps; + private RangeItem[] _virtualRangeOverlaps; private readonly ConcurrentQueue _deferredUnmaps; private int _hasDeferredUnmaps; @@ -75,7 +75,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { _memoryManager = memoryManager; _virtualRanges = []; - _virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity]; + _virtualRangeOverlaps = new RangeItem[BufferCache.OverlapsBufferInitialCapacity]; _deferredUnmaps = new ConcurrentQueue(); } @@ -106,18 +106,17 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if the range already existed, false if a new one was created and added public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range) { - VirtualRange[] overlaps = _virtualRangeOverlaps; - int overlapsCount; - + OverlapResult result; + if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0) { while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange)) { - overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps); + result = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size); - for (int index = 0; index < overlapsCount; index++) + if (result.StartIndex >= 0) { - _virtualRanges.Remove(overlaps[index]); + _virtualRanges.RemoveRange(result); } } } @@ -126,27 +125,23 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong originalVa = gpuVa; - overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps); - - if (overlapsCount != 0) + _virtualRanges.Lock.EnterWriteLock(); + result = _virtualRanges.FindOverlapsNonOverlappingAsSpan(gpuVa, size, out ReadOnlySpan> overlaps); + + if (overlaps.Length != 0) { // The virtual range already exists. We just need to check if our range fits inside // the existing one, and if not, we must extend the existing one. ulong endAddress = gpuVa + size; - VirtualRange overlap0 = overlaps[0]; + VirtualRange overlap0 = overlaps[0].Value; if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress) { - for (int index = 0; index < overlapsCount; index++) - { - VirtualRange virtualRange = overlaps[index]; - - gpuVa = Math.Min(gpuVa, virtualRange.Address); - endAddress = Math.Max(endAddress, virtualRange.EndAddress); - - _virtualRanges.Remove(virtualRange); - } + gpuVa = Math.Min(gpuVa, overlaps[0].Address); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); + + _virtualRanges.RemoveRange(result); ulong newSize = endAddress - gpuVa; MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize); @@ -170,6 +165,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _virtualRanges.Add(virtualRange); } + _virtualRanges.Lock.ExitWriteLock(); ShrinkOverlapsBufferIfNeeded(); diff --git a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs index 894078aee..da3e7983f 100644 --- a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs +++ b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs @@ -19,16 +19,18 @@ namespace Ryujinx.Memory.Range /// Start address of the search region /// Size of the search region /// Factory for creating new ranges - public void GetOrAddRegions(List list, ulong address, ulong size, Func factory) + public void GetOrAddRegions(out List list, ulong address, ulong size, Func factory) { // (regarding the specific case this generalized function is used for) // A new region may be split into multiple parts if multiple virtual regions have mapped to it. // For instance, while a virtual mapping could cover 0-2 in physical space, the space 0-1 may have already been reserved... // So we need to return both the split 0-1 and 1-2 ranges. - T[] results = new T[1]; - int count = FindOverlapsNonOverlapping(address, size, ref results); - + RangeItem[] results = new RangeItem[1]; + int count = FindOverlapsNonOverlapping(address, size, ref results).Count; + list = new List(count); + Lock.EnterWriteLock(); + if (count == 0) { // The region is fully unmapped. Create and add it to the range list. @@ -43,11 +45,12 @@ namespace Ryujinx.Memory.Range for (int i = 0; i < count; i++) { - T region = results[i]; + T region = results[i].Value; if (count == 1 && region.Address == address && region.Size == size) { // Exact match, no splitting required. list.Add(region); + Lock.ExitWriteLock(); return; } @@ -85,6 +88,8 @@ namespace Ryujinx.Memory.Range Add(fillRegion); } } + + Lock.ExitWriteLock(); } /// diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index 72cef1de0..980fe0ad8 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -1,45 +1,68 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; using System.Runtime.CompilerServices; +using System.Threading; namespace Ryujinx.Memory.Range { + public readonly struct RangeItem where TValue : IRange + { + public readonly ulong Address; + public readonly ulong EndAddress; + + public readonly TValue Value; + + public RangeItem(TValue value) + { + Value = value; + + Address = value.Address; + EndAddress = value.Address + value.Size; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool OverlapsWith(ulong address, ulong endAddress) + { + return Address < endAddress && address < EndAddress; + } + } + /// + /// Result of an Overlaps Finder function. + /// + /// + /// startIndex is inclusive. + /// endIndex is exclusive. + /// + public readonly struct OverlapResult + { + public readonly int StartIndex = -1; + public readonly int EndIndex = -1; + public int Count => EndIndex - StartIndex; + + public OverlapResult(int startIndex, int endIndex) + { + this.StartIndex = startIndex; + this.EndIndex = endIndex; + } + } + /// /// Sorted list of ranges that supports binary search. /// /// Type of the range. public class RangeList : IEnumerable where T : IRange { - private readonly struct RangeItem where TValue : IRange - { - public readonly ulong Address; - public readonly ulong EndAddress; - - public readonly TValue Value; - - public RangeItem(TValue value) - { - Value = value; - - Address = value.Address; - EndAddress = value.Address + value.Size; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool OverlapsWith(ulong address, ulong endAddress) - { - return Address < endAddress && address < EndAddress; - } - } - private const int BackingInitialSize = 1024; - private const int ArrayGrowthSize = 32; private RangeItem[] _items; private readonly int _backingGrowthSize; public int Count { get; protected set; } + + public readonly ReaderWriterLockSlim Lock = new ReaderWriterLockSlim(); /// /// Creates a new range list. @@ -74,15 +97,10 @@ namespace Ryujinx.Memory.Range /// True if the item was located and updated, false otherwise public bool Update(T item) { - int index = BinarySearch(item.Address); + int index = BinarySearchLeftEdge(item.Address); if (index >= 0) { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - while (index < Count) { if (_items[index].Value.Equals(item)) @@ -129,7 +147,7 @@ namespace Ryujinx.Memory.Range } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void RemoveAt(int index) + public void RemoveAt(int index) { if (index < --Count) { @@ -137,6 +155,32 @@ namespace Ryujinx.Memory.Range } } + public void RemoveRange(int start, int end) + { + if (end <= Count) + { + Array.Copy(_items, end, _items, start, Count - end); + Count -= end - start; + } + else if (end == Count) + { + Count = start; + } + } + + public void RemoveRange(OverlapResult overlapResult) + { + if (overlapResult.EndIndex < Count) + { + Array.Copy(_items, overlapResult.EndIndex, _items, overlapResult.StartIndex, Count - overlapResult.EndIndex); + Count -= overlapResult.Count; + } + else if (overlapResult.EndIndex == Count) + { + Count = overlapResult.StartIndex; + } + } + /// /// Removes an item from the list. /// @@ -144,15 +188,10 @@ namespace Ryujinx.Memory.Range /// True if the item was removed, or false if it was not found public bool Remove(T item) { - int index = BinarySearch(item.Address); + int index = BinarySearchLeftEdge(item.Address); if (index >= 0) { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - while (index < Count) { if (_items[index].Value.Equals(item)) @@ -180,15 +219,10 @@ namespace Ryujinx.Memory.Range /// The item to be updated public void UpdateEndAddress(T item) { - int index = BinarySearch(item.Address); + int index = BinarySearchLeftEdge(item.Address); if (index >= 0) { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - while (index < Count) { if (_items[index].Value.Equals(item)) @@ -250,7 +284,7 @@ namespace Ryujinx.Memory.Range /// Item to check for overlaps /// Output array where matches will be written. It is automatically resized to fit the results /// The number of overlapping items found - public int FindOverlaps(T item, ref T[] output) + public OverlapResult FindOverlaps(T item, ref RangeItem[] output) { return FindOverlaps(item.Address, item.Size, ref output); } @@ -262,33 +296,42 @@ namespace Ryujinx.Memory.Range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results /// The number of overlapping items found - public int FindOverlaps(ulong address, ulong size, ref T[] output) + public OverlapResult FindOverlaps(ulong address, ulong size, ref RangeItem[] output) { int outputIndex = 0; ulong endAddress = address + size; + int endIndex = -1; + for (int i = 0; i < Count; i++) { ref RangeItem item = ref _items[i]; if (item.Address >= endAddress) { + endIndex = i; break; } if (item.OverlapsWith(address, endAddress)) { - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = item.Value; + outputIndex++; } } - return outputIndex; + if (endIndex == -1 && outputIndex > 0) + { + endIndex = Count; + } + + if (endIndex - outputIndex >= 0) + { + Array.Resize(ref output, outputIndex); + Array.Copy(_items, endIndex - outputIndex, output, 0, outputIndex); + } + + return new OverlapResult(endIndex - outputIndex, endIndex); } /// @@ -302,7 +345,7 @@ namespace Ryujinx.Memory.Range /// Item to check for overlaps /// Output array where matches will be written. It is automatically resized to fit the results /// The number of overlapping items found - public int FindOverlapsNonOverlapping(T item, ref T[] output) + public OverlapResult FindOverlapsNonOverlapping(T item, ref RangeItem[] output) { return FindOverlapsNonOverlapping(item.Address, item.Size, ref output); } @@ -319,36 +362,64 @@ namespace Ryujinx.Memory.Range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results /// The number of overlapping items found - public int FindOverlapsNonOverlapping(ulong address, ulong size, ref T[] output) + public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size, ref RangeItem[] output) { // This is a bit faster than FindOverlaps, but only works // when none of the items on the list overlaps with each other. - int outputIndex = 0; ulong endAddress = address + size; - int index = BinarySearch(address, endAddress); + (int index, int endIndex) = BinarySearchEdges(address, endAddress); if (index >= 0) { - while (index > 0 && _items[index - 1].OverlapsWith(address, endAddress)) - { - index--; - } - - do - { - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = _items[index++].Value; - } - while (index < Count && _items[index].OverlapsWith(address, endAddress)); + Array.Resize(ref output, endIndex - index); + Array.Copy(_items, index, output, 0, endIndex - index); } - return outputIndex; + return new OverlapResult(index, endIndex); + } + + public OverlapResult FindOverlapsNonOverlappingAsSpan(ulong address, ulong size, out ReadOnlySpan> span) + { + // This is a bit faster than FindOverlaps, but only works + // when none of the items on the list overlaps with each other. + + ulong endAddress = address + size; + + (int index, int endIndex) = BinarySearchEdges(address, endAddress); + + if (index >= 0) + { + span = new ReadOnlySpan>(_items, index, endIndex - index); + return new OverlapResult(index, endIndex); + } + + span = ReadOnlySpan>.Empty; + return new OverlapResult(index, endIndex); + } + + /// + /// Gets range of all items on the list overlapping the specified memory range. + /// + /// + /// This method only returns correct results if none of the items on the list overlaps with + /// each other. If that is not the case, this method should not be used. + /// This method is faster than the regular method to find all overlaps. + /// + /// Start address of the range + /// Size in bytes of the range + /// Range information of overlapping items found + public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size) + { + // This is a bit faster than FindOverlaps, but only works + // when none of the items on the list overlaps with each other. + + ulong endAddress = address + size; + + (int index, int endIndex) = BinarySearchEdges(address, endAddress); + + return new OverlapResult(index, endIndex); } /// @@ -357,38 +428,17 @@ namespace Ryujinx.Memory.Range /// Address to find /// Output array where matches will be written. It is automatically resized to fit the results /// The number of matches found - public int FindOverlaps(ulong address, ref T[] output) + public OverlapResult FindOverlaps(ulong address, ref RangeItem[] output) { - int index = BinarySearch(address); - - int outputIndex = 0; + (int index, int endIndex) = BinarySearchEdges(address); if (index >= 0) { - while (index > 0 && _items[index - 1].Address == address) - { - index--; - } - - while (index < Count) - { - ref RangeItem overlap = ref _items[index++]; - - if (overlap.Address != address) - { - break; - } - - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = overlap.Value; - } + Array.Resize(ref output, endIndex - index); + Array.Copy(_items, index, output, 0, endIndex - index); } - return outputIndex; + return new OverlapResult(index, endIndex); } /// @@ -427,6 +477,202 @@ namespace Ryujinx.Memory.Range return ~left; } + /// + /// Performs binary search on the internal list of items. + /// + /// Address to find + /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + private int BinarySearchLeftEdge(ulong address) + { + if (Count == 0) + return ~0; + + if (Count == 1) + { + ref RangeItem item = ref _items[0]; + + if (address == item.Address) + { + return 0; + } + if (address < item.Address) + { + return ~0; + } + else + { + return ~1; + } + } + + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.Address == address; + + if (range == 0) + { + if (match) + return middle; + else if (address < item.Address) + return ~(right); + else + return ~(right + 1); + } + + if (match) + { + right = middle; + } + else if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + /// + /// Performs binary search on the internal list of items. + /// + /// Start address of the range + /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + private (int, int) BinarySearchEdges(ulong address) + { + if (Count == 0) + return (~0, ~0); + + if (Count == 1) + { + ref RangeItem item = ref _items[0]; + + if (item.Address == address) + { + return (0, 1); + } + + if (address < item.Address) + { + return (~0, ~0); + } + else + { + return (~1, ~1); + } + } + + int left = 0; + int right = Count - 1; + + int leftEdge = -1; + int rightEdgeMatch = -1; + int rightEdgeNoMatch = -1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.Address == address; + + if (range == 0) + { + if (match) + { + leftEdge = middle; + break; + } + else if (address < item.Address) + { + return (~right, ~right); + } + else + { + return (~(right + 1), ~(right + 1)); + } + } + + if (match) + { + right = middle; + if (rightEdgeMatch == -1) + rightEdgeMatch = middle; + } + else if (address < item.Address) + { + right = middle - 1; + rightEdgeNoMatch = middle; + } + else + { + left = middle + 1; + } + } + + if (left > right) + { + return (~left, ~left); + } + + if (rightEdgeMatch == -1) + { + return (leftEdge, leftEdge + 1); + } + + left = rightEdgeMatch; + right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = right - (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.Address == address; + + if (range == 0) + { + if (match) + return (leftEdge, middle + 1); + else + return (leftEdge, middle); + } + + if (match) + { + left = middle; + } + else if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return (leftEdge, right + 1); + } + /// /// Performs binary search for items overlapping a given memory range. /// @@ -464,6 +710,205 @@ namespace Ryujinx.Memory.Range return ~left; } + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + private int BinarySearchLeftEdge(ulong address, ulong endAddress) + { + if (Count == 0) + return ~0; + + if (Count == 1) + { + ref RangeItem item = ref _items[0]; + + if (item.OverlapsWith(address, endAddress)) + { + return 0; + } + + if (address < item.Address) + { + return ~0; + } + else + { + return ~1; + } + } + + int left = 0; + int right = Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + return middle; + else if (address < item.Address) + return ~(right); + else + return ~(right + 1); + } + + if (match) + { + right = middle; + } + else if(address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + private (int, int) BinarySearchEdges(ulong address, ulong endAddress) + { + if (Count == 0) + return (~0, ~0); + + if (Count == 1) + { + ref RangeItem item = ref _items[0]; + + if (item.OverlapsWith(address, endAddress)) + { + return (0, 1); + } + + if (address < item.Address) + { + return (~0, ~0); + } + else + { + return (~1, ~1); + } + } + + int left = 0; + int right = Count - 1; + + int leftEdge = -1; + int rightEdgeMatch = -1; + int rightEdgeNoMatch = -1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + { + leftEdge = middle; + break; + } + else if (address < item.Address) + { + return (~right, ~right); + } + else + { + return (~(right + 1), ~(right + 1)); + } + } + + if (match) + { + right = middle; + if (rightEdgeMatch == -1) + rightEdgeMatch = middle; + } + else if (address < item.Address) + { + right = middle - 1; + rightEdgeNoMatch = middle; + } + else + { + left = middle + 1; + } + } + + if (left > right) + { + return (~left, ~left); + } + + if (rightEdgeMatch == -1) + { + return (leftEdge, leftEdge + 1); + } + + left = rightEdgeMatch; + right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = right - (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + return (leftEdge, middle + 1); + else + return (leftEdge, middle); + } + + if (match) + { + left = middle; + } + else if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return (leftEdge, right + 1); + } + public IEnumerator GetEnumerator() { for (int i = 0; i < Count; i++) diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index e7791fec3..bc69aea3d 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -1,5 +1,6 @@ using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; +using System; using System.Collections.Generic; namespace Ryujinx.Memory.Tracking @@ -76,17 +77,15 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - - int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + regions.Lock.EnterReadLock(); + regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan> overlaps); + + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = overlaps[i].Value; // If the region has been fully remapped, signal that it has been mapped again. bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); @@ -97,6 +96,7 @@ namespace Ryujinx.Memory.Tracking region.UpdateProtection(); } + regions.Lock.ExitReadLock(); } } } @@ -114,20 +114,19 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; + regions.Lock.EnterReadLock(); + regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan> overlaps); - int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = overlaps[i].Value; region.SignalMappingChanged(false); } + regions.Lock.ExitReadLock(); } } } @@ -165,10 +164,11 @@ namespace Ryujinx.Memory.Tracking /// A list of virtual regions within the given range internal List GetVirtualRegionsForHandle(ulong va, ulong size, bool guest) { - List result = []; NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; - regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); - + regions.Lock.EnterUpgradeableReadLock(); + regions.GetOrAddRegions(out List result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); + regions.Lock.ExitUpgradeableReadLock(); + return result; } @@ -296,25 +296,25 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; + ref RangeItem[] overlaps = ref ThreadStaticArray>.Get(); + + regions.Lock.EnterReadLock(); + OverlapResult result = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); + regions.Lock.ExitReadLock(); - int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); - - if (count == 0 && !precise) + if (overlaps.Length == 0 && !precise) { if (_memoryManager.IsRangeMapped(address, size)) { // TODO: There is currently the possibility that a page can be protected after its virtual region is removed. // This code handles that case when it happens, but it would be better to find out how this happens. _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest); + return true; // This memory _should_ be mapped, so we need to try again. } - else - { - shouldThrow = true; - } + + shouldThrow = true; } else { @@ -324,9 +324,9 @@ namespace Ryujinx.Memory.Tracking size += (ulong)_pageSize; } - for (int i = 0; i < count; i++) + for (int i = 0; i < result.Count; i++) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = overlaps[i].Value; if (precise) { From ce438f7276b7d30c1d20ecc8fcd0df0f6dccdb77 Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Mon, 2 Jun 2025 01:17:34 +0200 Subject: [PATCH 2/7] implement quickAccess for fast look-up some values are reused often and will be quicker to access by first checking an array of the last used items. removed and renamed some older functions. --- .../Memory/BufferCache.cs | 12 +- src/Ryujinx.Memory/Range/RangeList.cs | 257 ++++++------------ 2 files changed, 85 insertions(+), 184 deletions(-) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index 41d2af024..6c2520979 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -335,7 +335,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; ulong alignedSize = alignedEndAddress - alignedAddress; - Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); + Buffer buffer = _buffers.FindOverlapFast(alignedAddress, alignedSize); BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); @@ -402,7 +402,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (subRange.Address != MemoryManager.PteUnmapped) { - Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); + Buffer buffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size); virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); physicalBuffers.Add(buffer); @@ -890,7 +890,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { MemoryRange subRange = range.GetSubRange(i); - Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); + Buffer subBuffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size); subBuffer.SynchronizeMemory(subRange.Address, subRange.Size); @@ -938,7 +938,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (size != 0) { - buffer = _buffers.FindFirstOverlap(address, size); + buffer = _buffers.FindOverlapFast(address, size); buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); @@ -950,7 +950,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - buffer = _buffers.FindFirstOverlap(address, 1); + buffer = _buffers.FindOverlapFast(address, 1); } return buffer; @@ -988,7 +988,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (size != 0) { - Buffer buffer = _buffers.FindFirstOverlap(address, size); + Buffer buffer = _buffers.FindOverlapFast(address, size); if (copyBackVirtual) { diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index 980fe0ad8..c2e39b576 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -1,27 +1,17 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; using System.Threading; namespace Ryujinx.Memory.Range { - public readonly struct RangeItem where TValue : IRange + public readonly struct RangeItem(TValue value) where TValue : IRange { - public readonly ulong Address; - public readonly ulong EndAddress; + public readonly ulong Address = value.Address; + public readonly ulong EndAddress = value.Address + value.Size; - public readonly TValue Value; - - public RangeItem(TValue value) - { - Value = value; - - Address = value.Address; - EndAddress = value.Address + value.Size; - } + public readonly TValue Value = value; [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool OverlapsWith(ulong address, ulong endAddress) @@ -29,6 +19,7 @@ namespace Ryujinx.Memory.Range return Address < endAddress && address < EndAddress; } } + /// /// Result of an Overlaps Finder function. /// @@ -62,7 +53,12 @@ namespace Ryujinx.Memory.Range public int Count { get; protected set; } - public readonly ReaderWriterLockSlim Lock = new ReaderWriterLockSlim(); + public readonly ReaderWriterLockSlim Lock = new(); + + private const int QuickAccessLength = 8; + private int _offset; + private int _count; + private RangeItem[] _quickAccess = new RangeItem[QuickAccessLength]; /// /// Creates a new range list. @@ -95,7 +91,7 @@ namespace Ryujinx.Memory.Range /// /// The item to be updated /// True if the item was located and updated, false otherwise - public bool Update(T item) + protected bool Update(T item) { int index = BinarySearchLeftEdge(item.Address); @@ -106,6 +102,10 @@ namespace Ryujinx.Memory.Range if (_items[index].Value.Equals(item)) { _items[index] = new RangeItem(item); + + _quickAccess = new RangeItem[QuickAccessLength]; + _count = 0; + _offset = 0; return true; } @@ -147,27 +147,19 @@ namespace Ryujinx.Memory.Range } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void RemoveAt(int index) + protected void RemoveAt(int index) { if (index < --Count) { Array.Copy(_items, index + 1, _items, index, Count - index); } - } - public void RemoveRange(int start, int end) - { - if (end <= Count) - { - Array.Copy(_items, end, _items, start, Count - end); - Count -= end - start; - } - else if (end == Count) - { - Count = start; - } + _quickAccess = new RangeItem[QuickAccessLength]; + _count = 0; + _offset = 0; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void RemoveRange(OverlapResult overlapResult) { if (overlapResult.EndIndex < Count) @@ -179,6 +171,10 @@ namespace Ryujinx.Memory.Range { Count = overlapResult.StartIndex; } + + _quickAccess = new RangeItem[QuickAccessLength]; + _count = 0; + _offset = 0; } /// @@ -214,62 +210,39 @@ namespace Ryujinx.Memory.Range } /// - /// Updates an item's end address. - /// - /// The item to be updated - public void UpdateEndAddress(T item) - { - int index = BinarySearchLeftEdge(item.Address); - - if (index >= 0) - { - while (index < Count) - { - if (_items[index].Value.Equals(item)) - { - _items[index] = new RangeItem(item); - - return; - } - - if (_items[index].Address > item.Address) - { - break; - } - - index++; - } - } - } - - /// - /// Gets the first item on the list overlapping in memory with the specified item. + /// Gets an item on the list overlapping the specified memory range. /// /// - /// Despite the name, this has no ordering guarantees of the returned item. - /// It only ensures that the item returned overlaps the specified item. - /// - /// Item to check for overlaps - /// The overlapping item, or the default value for the type if none found - public T FindFirstOverlap(T item) - { - return FindFirstOverlap(item.Address, item.Size); - } - - /// - /// Gets the first item on the list overlapping the specified memory range. - /// - /// - /// Despite the name, this has no ordering guarantees of the returned item. + /// This has no ordering guarantees of the returned item. /// It only ensures that the item returned overlaps the specified memory range. /// /// Start address of the range /// Size in bytes of the range /// The overlapping item, or the default value for the type if none found - public T FindFirstOverlap(ulong address, ulong size) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T FindOverlapFast(ulong address, ulong size) { + for (int i = 0; i < _quickAccess.Length; i++) + { + ref RangeItem item = ref _quickAccess[(i + _offset) % _quickAccess.Length]; + + if (item.OverlapsWith(address, address + size)) + { + return item.Value; + } + } + int index = BinarySearch(address, address + size); + if (_count < _quickAccess.Length) + { + _quickAccess[_count++] = _items[index]; + } + else + { + _quickAccess[_offset++ % _quickAccess.Length] = _items[index]; + } + if (index < 0) { return default; @@ -277,18 +250,7 @@ namespace Ryujinx.Memory.Range return _items[index].Value; } - - /// - /// Gets all items overlapping with the specified item in memory. - /// - /// Item to check for overlaps - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public OverlapResult FindOverlaps(T item, ref RangeItem[] output) - { - return FindOverlaps(item.Address, item.Size, ref output); - } - + /// /// Gets all items on the list overlapping the specified memory range. /// @@ -301,10 +263,13 @@ namespace Ryujinx.Memory.Range int outputIndex = 0; ulong endAddress = address + size; - + + int startIndex = BinarySearchLeftEdge(address); + if (startIndex < 0) + startIndex = ~startIndex; int endIndex = -1; - for (int i = 0; i < Count; i++) + for (int i = startIndex; i < Count; i++) { ref RangeItem item = ref _items[i]; @@ -325,29 +290,26 @@ namespace Ryujinx.Memory.Range endIndex = Count; } - if (endIndex - outputIndex >= 0) + if (outputIndex > 0 && outputIndex == endIndex - startIndex) { Array.Resize(ref output, outputIndex); Array.Copy(_items, endIndex - outputIndex, output, 0, outputIndex); + + return new OverlapResult(startIndex, endIndex); } - - return new OverlapResult(endIndex - outputIndex, endIndex); - } - - /// - /// Gets all items overlapping with the specified item in memory. - /// - /// - /// This method only returns correct results if none of the items on the list overlaps with - /// each other. If that is not the case, this method should not be used. - /// This method is faster than the regular method to find all overlaps. - /// - /// Item to check for overlaps - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public OverlapResult FindOverlapsNonOverlapping(T item, ref RangeItem[] output) - { - return FindOverlapsNonOverlapping(item.Address, item.Size, ref output); + else if (outputIndex > 0) + { + Array.Resize(ref output, outputIndex); + int arrIndex = 0; + for (int i = startIndex; i < endIndex; i++) + { + output[arrIndex++] = _items[i]; + } + + return new OverlapResult(endIndex - outputIndex, endIndex); + } + + return new OverlapResult(); } /// @@ -362,6 +324,7 @@ namespace Ryujinx.Memory.Range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results /// The number of overlapping items found + [MethodImpl(MethodImplOptions.AggressiveInlining)] public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size, ref RangeItem[] output) { // This is a bit faster than FindOverlaps, but only works @@ -380,6 +343,7 @@ namespace Ryujinx.Memory.Range return new OverlapResult(index, endIndex); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public OverlapResult FindOverlapsNonOverlappingAsSpan(ulong address, ulong size, out ReadOnlySpan> span) { // This is a bit faster than FindOverlaps, but only works @@ -410,6 +374,7 @@ namespace Ryujinx.Memory.Range /// Start address of the range /// Size in bytes of the range /// Range information of overlapping items found + [MethodImpl(MethodImplOptions.AggressiveInlining)] public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size) { // This is a bit faster than FindOverlaps, but only works @@ -428,6 +393,7 @@ namespace Ryujinx.Memory.Range /// Address to find /// Output array where matches will be written. It is automatically resized to fit the results /// The number of matches found + [MethodImpl(MethodImplOptions.AggressiveInlining)] public OverlapResult FindOverlaps(ulong address, ref RangeItem[] output) { (int index, int endIndex) = BinarySearchEdges(address); @@ -446,6 +412,7 @@ namespace Ryujinx.Memory.Range /// /// Address to find /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int BinarySearch(ulong address) { int left = 0; @@ -482,6 +449,7 @@ namespace Ryujinx.Memory.Range /// /// Address to find /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int BinarySearchLeftEdge(ulong address) { if (Count == 0) @@ -550,6 +518,7 @@ namespace Ryujinx.Memory.Range /// /// Start address of the range /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] private (int, int) BinarySearchEdges(ulong address) { if (Count == 0) @@ -679,6 +648,7 @@ namespace Ryujinx.Memory.Range /// Start address of the range /// End address of the range /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int BinarySearch(ulong address, ulong endAddress) { int left = 0; @@ -716,76 +686,7 @@ namespace Ryujinx.Memory.Range /// Start address of the range /// End address of the range /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list - private int BinarySearchLeftEdge(ulong address, ulong endAddress) - { - if (Count == 0) - return ~0; - - if (Count == 1) - { - ref RangeItem item = ref _items[0]; - - if (item.OverlapsWith(address, endAddress)) - { - return 0; - } - - if (address < item.Address) - { - return ~0; - } - else - { - return ~1; - } - } - - int left = 0; - int right = Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref _items[middle]; - - bool match = item.OverlapsWith(address, endAddress); - - if (range == 0) - { - if (match) - return middle; - else if (address < item.Address) - return ~(right); - else - return ~(right + 1); - } - - if (match) - { - right = middle; - } - else if(address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return ~left; - } - - /// - /// Performs binary search for items overlapping a given memory range. - /// - /// Start address of the range - /// End address of the range - /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] private (int, int) BinarySearchEdges(ulong address, ulong endAddress) { if (Count == 0) From fe5b6ad7c75abb808de11e64c5c719f6021ef6f4 Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Tue, 3 Jun 2025 23:53:48 +0200 Subject: [PATCH 3/7] refactor: use Spans for aligned buffer creation --- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 15 +++---- .../Memory/BufferBackingState.cs | 6 +-- .../Memory/BufferCache.cs | 39 ++++++++++--------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index 25034b95a..060358d62 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong size, BufferStage stage, bool sparseCompatible, - IEnumerable> baseBuffers = null) + ReadOnlySpan> baseBuffers) { _context = context; _physicalMemory = physicalMemory; @@ -126,21 +126,22 @@ namespace Ryujinx.Graphics.Gpu.Memory _useGranular = size > GranularBufferThreshold; - IEnumerable baseHandles = null; + List baseHandles = null; - if (baseBuffers != null) + if (!baseBuffers.IsEmpty) { - baseHandles = baseBuffers.SelectMany(buffer => + baseHandles = new List(); + foreach (RangeItem buffer in baseBuffers) { if (buffer.Value._useGranular) { - return buffer.Value._memoryTrackingGranular.GetHandles(); + baseHandles.AddRange((buffer.Value._memoryTrackingGranular.GetHandles())); } else { - return Enumerable.Repeat(buffer.Value._memoryTracking, 1); + baseHandles.Add(buffer.Value._memoryTracking); } - }); + } } if (_useGranular) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs index 61f0f9d6d..1029c38d9 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs @@ -57,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Parent buffer /// Initial buffer stage /// Buffers to inherit state from - public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable> baseBuffers = null) + public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, ReadOnlySpan> baseBuffers) { _size = (int)parent.Size; _systemMemoryType = context.Capabilities.MemoryType; @@ -73,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory BufferStage storageFlags = stage & BufferStage.StorageMask; - if (parent.Size > DeviceLocalSizeThreshold && baseBuffers == null) + if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.IsEmpty) { _desiredType = BufferBackingType.DeviceMemory; } @@ -101,7 +101,7 @@ namespace Ryujinx.Graphics.Gpu.Memory // TODO: Might be nice to force atomic access to be device local for any stage. } - if (baseBuffers != null) + if (!baseBuffers.IsEmpty) { foreach (RangeItem buffer in baseBuffers) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index 6c2520979..6afcb5a68 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -495,8 +495,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) { _buffers.Lock.EnterWriteLock(); - RangeItem[] overlaps = _bufferOverlaps; - OverlapResult result = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); if (result.Count != 0) { @@ -534,10 +533,9 @@ namespace Ryujinx.Graphics.Gpu.Memory size = Math.Max(size, growthSize); endAddress = address + size; - result = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out overlaps); } - _buffers.RemoveRange(result); address = Math.Min(address, overlaps[0].Address); endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); @@ -549,15 +547,20 @@ namespace Ryujinx.Graphics.Gpu.Memory anySparseCompatible |= buffer.Value.SparseCompatible; } + ulong newSize = endAddress - address; - CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, result.Count); + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); + + _buffers.RemoveRange(result); + + _buffers.Add(newBuffer); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false, ReadOnlySpan>.Empty); _buffers.Add(buffer); } @@ -579,8 +582,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment) { _buffers.Lock.EnterWriteLock(); - RangeItem[] overlaps = _bufferOverlaps; - OverlapResult result = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); bool sparseAligned = alignment >= SparseBufferAlignmentSize; if (result.Count != 0) @@ -611,21 +613,23 @@ namespace Ryujinx.Graphics.Gpu.Memory address &= ~(alignment - 1); oldOverlapsCount = result.Count; - result = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps); + result = _buffers.FindOverlapsNonOverlappingAsSpan(address, endAddress - address, out overlaps); } while (oldOverlapsCount != result.Count); - _buffers.RemoveRange(result); - ulong newSize = endAddress - address; - CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, result.Count); + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); + + _buffers.RemoveRange(result); + + _buffers.Add(newBuffer); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned, ReadOnlySpan>.Empty); _buffers.Add(buffer); } @@ -644,14 +648,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers overlapping the range - /// Total of overlaps - private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, RangeItem[] overlaps, int overlapsCount) + private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, ReadOnlySpan> overlaps) { Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); - _buffers.Add(newBuffer); - - for (int index = 0; index < overlapsCount; index++) + for (int index = 0; index < overlaps.Length; index++) { Buffer buffer = overlaps[index].Value; @@ -669,6 +670,8 @@ namespace Ryujinx.Graphics.Gpu.Memory NotifyBuffersModified?.Invoke(); RecreateMultiRangeBuffers(address, size); + + return newBuffer; } /// From d7a6f95c8b04f5ee0f56093924481807a318712c Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Wed, 4 Jun 2025 00:54:47 +0200 Subject: [PATCH 4/7] add comments --- src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs | 2 ++ src/Ryujinx.Memory/Tracking/MemoryTracking.cs | 1 + 2 files changed, 3 insertions(+) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index 1e712b542..ef027667e 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -289,6 +289,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { RangeItem[] overlaps = new RangeItem[1]; + // We use the non-span method here because keeping the lock will cause a deadlock. Lock.EnterReadLock(); OverlapResult result = FindOverlapsNonOverlapping(address, size, ref overlaps); Lock.ExitReadLock(); @@ -409,6 +410,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { + // We use the non-span method here because the array is partially modified by the code, which would invalidate a span. rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps).Count; } Lock.ExitReadLock(); diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index bc69aea3d..7aecd7f4d 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -299,6 +299,7 @@ namespace Ryujinx.Memory.Tracking NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; ref RangeItem[] overlaps = ref ThreadStaticArray>.Get(); + // We use the non-span method here because keeping the lock will cause a deadlock. regions.Lock.EnterReadLock(); OverlapResult result = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); regions.Lock.ExitReadLock(); From 14a542ec5fbe532e3e1d2370b280507b8ccebafb Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Wed, 4 Jun 2025 01:03:03 +0200 Subject: [PATCH 5/7] Update BufferModifiedRangeList.cs --- src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index ef027667e..a31156121 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -2,7 +2,6 @@ using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System; using System.Linq; -using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { @@ -417,7 +416,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (rangeCount == -1) { - _migrationTarget.WaitForAndFlushRanges(address, size); + _migrationTarget!.WaitForAndFlushRanges(address, size); return; } From bc37866ac946bc9c7b72970c341ffadfb1086e8b Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Wed, 4 Jun 2025 02:13:31 +0200 Subject: [PATCH 6/7] remove duplicate function BinarySearchLeftEdge(ulong address) matches BinarySearch(ulong address), so the former is not needed --- src/Ryujinx.Memory/Range/RangeList.cs | 75 ++------------------------- 1 file changed, 3 insertions(+), 72 deletions(-) diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index c2e39b576..a1df8f142 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -93,7 +93,7 @@ namespace Ryujinx.Memory.Range /// True if the item was located and updated, false otherwise protected bool Update(T item) { - int index = BinarySearchLeftEdge(item.Address); + int index = BinarySearch(item.Address); if (index >= 0) { @@ -184,7 +184,7 @@ namespace Ryujinx.Memory.Range /// True if the item was removed, or false if it was not found public bool Remove(T item) { - int index = BinarySearchLeftEdge(item.Address); + int index = BinarySearch(item.Address); if (index >= 0) { @@ -264,7 +264,7 @@ namespace Ryujinx.Memory.Range ulong endAddress = address + size; - int startIndex = BinarySearchLeftEdge(address); + int startIndex = BinarySearch(address); if (startIndex < 0) startIndex = ~startIndex; int endIndex = -1; @@ -444,75 +444,6 @@ namespace Ryujinx.Memory.Range return ~left; } - /// - /// Performs binary search on the internal list of items. - /// - /// Address to find - /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int BinarySearchLeftEdge(ulong address) - { - if (Count == 0) - return ~0; - - if (Count == 1) - { - ref RangeItem item = ref _items[0]; - - if (address == item.Address) - { - return 0; - } - if (address < item.Address) - { - return ~0; - } - else - { - return ~1; - } - } - - int left = 0; - int right = Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref _items[middle]; - - bool match = item.Address == address; - - if (range == 0) - { - if (match) - return middle; - else if (address < item.Address) - return ~(right); - else - return ~(right + 1); - } - - if (match) - { - right = middle; - } - else if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return ~left; - } - /// /// Performs binary search on the internal list of items. /// From 373fa30535bfc8fa2d5d12fe2cee319832904bbc Mon Sep 17 00:00:00 2001 From: LotP1 <68976644+LotP1@users.noreply.github.com> Date: Wed, 4 Jun 2025 02:24:00 +0200 Subject: [PATCH 7/7] update comments and remove unused functions --- src/Ryujinx.Memory/Range/RangeList.cs | 159 +------------------------- 1 file changed, 5 insertions(+), 154 deletions(-) diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index a1df8f142..d575f4583 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -257,7 +257,7 @@ namespace Ryujinx.Memory.Range /// Start address of the range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found + /// Range information of overlapping items found public OverlapResult FindOverlaps(ulong address, ulong size, ref RangeItem[] output) { int outputIndex = 0; @@ -323,7 +323,7 @@ namespace Ryujinx.Memory.Range /// Start address of the range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found + /// Range information of overlapping items found [MethodImpl(MethodImplOptions.AggressiveInlining)] public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size, ref RangeItem[] output) { @@ -364,7 +364,7 @@ namespace Ryujinx.Memory.Range } /// - /// Gets range of all items on the list overlapping the specified memory range. + /// Gets the range of all items on the list overlapping the specified memory range. /// /// /// This method only returns correct results if none of the items on the list overlaps with @@ -387,26 +387,6 @@ namespace Ryujinx.Memory.Range return new OverlapResult(index, endIndex); } - /// - /// Gets all items on the list with the specified memory address. - /// - /// Address to find - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of matches found - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public OverlapResult FindOverlaps(ulong address, ref RangeItem[] output) - { - (int index, int endIndex) = BinarySearchEdges(address); - - if (index >= 0) - { - Array.Resize(ref output, endIndex - index); - Array.Copy(_items, index, output, 0, endIndex - index); - } - - return new OverlapResult(index, endIndex); - } - /// /// Performs binary search on the internal list of items. /// @@ -443,136 +423,7 @@ namespace Ryujinx.Memory.Range return ~left; } - - /// - /// Performs binary search on the internal list of items. - /// - /// Start address of the range - /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private (int, int) BinarySearchEdges(ulong address) - { - if (Count == 0) - return (~0, ~0); - - if (Count == 1) - { - ref RangeItem item = ref _items[0]; - - if (item.Address == address) - { - return (0, 1); - } - - if (address < item.Address) - { - return (~0, ~0); - } - else - { - return (~1, ~1); - } - } - - int left = 0; - int right = Count - 1; - - int leftEdge = -1; - int rightEdgeMatch = -1; - int rightEdgeNoMatch = -1; - - while (left <= right) - { - int range = right - left; - - int middle = left + (range >> 1); - - ref RangeItem item = ref _items[middle]; - - bool match = item.Address == address; - - if (range == 0) - { - if (match) - { - leftEdge = middle; - break; - } - else if (address < item.Address) - { - return (~right, ~right); - } - else - { - return (~(right + 1), ~(right + 1)); - } - } - - if (match) - { - right = middle; - if (rightEdgeMatch == -1) - rightEdgeMatch = middle; - } - else if (address < item.Address) - { - right = middle - 1; - rightEdgeNoMatch = middle; - } - else - { - left = middle + 1; - } - } - - if (left > right) - { - return (~left, ~left); - } - - if (rightEdgeMatch == -1) - { - return (leftEdge, leftEdge + 1); - } - - left = rightEdgeMatch; - right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1; - - while (left <= right) - { - int range = right - left; - - int middle = right - (range >> 1); - - ref RangeItem item = ref _items[middle]; - - bool match = item.Address == address; - - if (range == 0) - { - if (match) - return (leftEdge, middle + 1); - else - return (leftEdge, middle); - } - - if (match) - { - left = middle; - } - else if (address < item.Address) - { - right = middle - 1; - } - else - { - left = middle + 1; - } - } - - return (leftEdge, right + 1); - } - + /// /// Performs binary search for items overlapping a given memory range. /// @@ -616,7 +467,7 @@ namespace Ryujinx.Memory.Range /// /// Start address of the range /// End address of the range - /// List index of the left-most item that overlaps, or complement index of nearest item with lower value on the list + /// Range information (inclusive, exclusive) of items that overlaps, or complement index of nearest item with lower value on the list [MethodImpl(MethodImplOptions.AggressiveInlining)] private (int, int) BinarySearchEdges(ulong address, ulong endAddress) {