diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index a2448d76f..060358d62 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong size, BufferStage stage, bool sparseCompatible, - IEnumerable baseBuffers = null) + ReadOnlySpan> baseBuffers) { _context = context; _physicalMemory = physicalMemory; @@ -126,21 +126,22 @@ namespace Ryujinx.Graphics.Gpu.Memory _useGranular = size > GranularBufferThreshold; - IEnumerable baseHandles = null; + List baseHandles = null; - if (baseBuffers != null) + if (!baseBuffers.IsEmpty) { - baseHandles = baseBuffers.SelectMany(buffer => + baseHandles = new List(); + foreach (RangeItem buffer in baseBuffers) { - if (buffer._useGranular) + if (buffer.Value._useGranular) { - return buffer._memoryTrackingGranular.GetHandles(); + baseHandles.AddRange((buffer.Value._memoryTrackingGranular.GetHandles())); } else { - return Enumerable.Repeat(buffer._memoryTracking, 1); + baseHandles.Add(buffer.Value._memoryTracking); } - }); + } } if (_useGranular) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs index a9b1f50e2..1029c38d9 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs @@ -1,4 +1,5 @@ using Ryujinx.Graphics.GAL; +using Ryujinx.Memory.Range; using System; using System.Collections.Generic; @@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Parent buffer /// Initial buffer stage /// Buffers to inherit state from - public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable baseBuffers = null) + public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, ReadOnlySpan> baseBuffers) { _size = (int)parent.Size; _systemMemoryType = context.Capabilities.MemoryType; @@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory BufferStage storageFlags = stage & BufferStage.StorageMask; - if (parent.Size > DeviceLocalSizeThreshold && baseBuffers == null) + if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.IsEmpty) { _desiredType = BufferBackingType.DeviceMemory; } @@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory // TODO: Might be nice to force atomic access to be device local for any stage. } - if (baseBuffers != null) + if (!baseBuffers.IsEmpty) { - foreach (Buffer buffer in baseBuffers) + foreach (RangeItem buffer in baseBuffers) { - CombineState(buffer.BackingState); + CombineState(buffer.Value.BackingState); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index d02efcb29..6afcb5a68 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -2,7 +2,6 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Memory.Range; using System; using System.Collections.Generic; -using System.Linq; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory @@ -42,7 +41,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly RangeList _buffers; private readonly MultiRangeList _multiRangeBuffers; - private Buffer[] _bufferOverlaps; + private RangeItem[] _bufferOverlaps; private readonly Dictionary _dirtyCache; private readonly Dictionary _modifiedCache; @@ -64,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _buffers = []; _multiRangeBuffers = []; - _bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity]; + _bufferOverlaps = new RangeItem[OverlapsBufferInitialCapacity]; _dirtyCache = new Dictionary(); @@ -79,7 +78,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Event arguments public void MemoryUnmappedHandler(object sender, UnmapEventArgs e) { - Buffer[] overlaps = new Buffer[10]; + RangeItem[] overlaps = new RangeItem[10]; int overlapCount; MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size); @@ -90,12 +89,12 @@ namespace Ryujinx.Graphics.Gpu.Memory lock (_buffers) { - overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps); + overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps).Count; } for (int i = 0; i < overlapCount; i++) { - overlaps[i].Unmapped(subRange.Address, subRange.Size); + overlaps[i].Value.Unmapped(subRange.Address, subRange.Size); } } } @@ -336,7 +335,7 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; ulong alignedSize = alignedEndAddress - alignedAddress; - Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); + Buffer buffer = _buffers.FindOverlapFast(alignedAddress, alignedSize); BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); @@ -403,7 +402,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (subRange.Address != MemoryManager.PteUnmapped) { - Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); + Buffer buffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size); virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); physicalBuffers.Add(buffer); @@ -495,10 +494,10 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) { - Buffer[] overlaps = _bufferOverlaps; - int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + _buffers.Lock.EnterWriteLock(); + OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); - if (overlapsCount != 0) + if (result.Count != 0) { // The buffer already exists. We can just return the existing buffer // if the buffer we need is fully contained inside the overlapping buffer. @@ -507,7 +506,7 @@ namespace Ryujinx.Graphics.Gpu.Memory // old buffer(s) to the new buffer. ulong endAddress = address + size; - Buffer overlap0 = overlaps[0]; + RangeItem overlap0 = overlaps[0]; if (overlap0.Address > address || overlap0.EndAddress < endAddress) { @@ -522,51 +521,51 @@ namespace Ryujinx.Graphics.Gpu.Memory // sequential memory. // Allowing for 2 pages (rather than just one) is necessary to catch cases where the // range crosses a page, and after alignment, ends having a size of 2 pages. - if (overlapsCount == 1 && + if (result.Count == 1 && address >= overlap0.Address && endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2) { // Try to grow the buffer by 1.5x of its current size. // This improves performance in the cases where the buffer is resized often by small amounts. - ulong existingSize = overlap0.Size; + ulong existingSize = overlap0.Value.Size; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; size = Math.Max(size, growthSize); endAddress = address + size; - overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out overlaps); } + + + address = Math.Min(address, overlaps[0].Address); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); - for (int index = 0; index < overlapsCount; index++) + for (int index = 0; index < result.Count; index++) { - Buffer buffer = overlaps[index]; + RangeItem buffer = overlaps[index]; - anySparseCompatible |= buffer.SparseCompatible; - - address = Math.Min(address, buffer.Address); - endAddress = Math.Max(endAddress, buffer.EndAddress); - - lock (_buffers) - { - _buffers.Remove(buffer); - } + anySparseCompatible |= buffer.Value.SparseCompatible; } + ulong newSize = endAddress - address; - CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, overlapsCount); + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps); + + _buffers.RemoveRange(result); + + _buffers.Add(newBuffer); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false, ReadOnlySpan>.Empty); - lock (_buffers) - { - _buffers.Add(buffer); - } + _buffers.Add(buffer); } + + _buffers.Lock.ExitWriteLock(); ShrinkOverlapsBufferIfNeeded(); } @@ -582,22 +581,22 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Alignment of the start address of the buffer private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment) { - Buffer[] overlaps = _bufferOverlaps; - int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); + _buffers.Lock.EnterWriteLock(); + OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); bool sparseAligned = alignment >= SparseBufferAlignmentSize; - if (overlapsCount != 0) + if (result.Count != 0) { // If the buffer already exists, make sure if covers the entire range, // and make sure it is properly aligned, otherwise sparse mapping may fail. ulong endAddress = address + size; - Buffer overlap0 = overlaps[0]; + RangeItem overlap0 = overlaps[0]; if (overlap0.Address > address || overlap0.EndAddress < endAddress || (overlap0.Address & (alignment - 1)) != 0 || - (!overlap0.SparseCompatible && sparseAligned)) + (!overlap0.Value.SparseCompatible && sparseAligned)) { // We need to make sure the new buffer is properly aligned. // However, after the range is aligned, it is possible that it @@ -605,47 +604,36 @@ namespace Ryujinx.Graphics.Gpu.Memory // and ensure we cover all overlaps. int oldOverlapsCount; + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); do { - for (int index = 0; index < overlapsCount; index++) - { - Buffer buffer = overlaps[index]; - - address = Math.Min(address, buffer.Address); - endAddress = Math.Max(endAddress, buffer.EndAddress); - } + address = Math.Min(address, overlaps[0].Address); address &= ~(alignment - 1); - oldOverlapsCount = overlapsCount; - overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps); - } - while (oldOverlapsCount != overlapsCount); - - lock (_buffers) - { - for (int index = 0; index < overlapsCount; index++) - { - _buffers.Remove(overlaps[index]); - } + oldOverlapsCount = result.Count; + result = _buffers.FindOverlapsNonOverlappingAsSpan(address, endAddress - address, out overlaps); } + while (oldOverlapsCount != result.Count); ulong newSize = endAddress - address; - CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, overlapsCount); + Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps); + + _buffers.RemoveRange(result); + + _buffers.Add(newBuffer); } } else { // No overlap, just create a new buffer. - Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned); + Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned, ReadOnlySpan>.Empty); - lock (_buffers) - { - _buffers.Add(buffer); - } + _buffers.Add(buffer); } + _buffers.Lock.ExitWriteLock(); ShrinkOverlapsBufferIfNeeded(); } @@ -660,19 +648,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The type of usage that created the buffer /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers overlapping the range - /// Total of overlaps - private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, Buffer[] overlaps, int overlapsCount) + private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, ReadOnlySpan> overlaps) { - Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps.Take(overlapsCount)); + Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps); - lock (_buffers) + for (int index = 0; index < overlaps.Length; index++) { - _buffers.Add(newBuffer); - } - - for (int index = 0; index < overlapsCount; index++) - { - Buffer buffer = overlaps[index]; + Buffer buffer = overlaps[index].Value; int dstOffset = (int)(buffer.Address - newBuffer.Address); @@ -688,6 +670,8 @@ namespace Ryujinx.Graphics.Gpu.Memory NotifyBuffersModified?.Invoke(); RecreateMultiRangeBuffers(address, size); + + return newBuffer; } /// @@ -909,7 +893,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { MemoryRange subRange = range.GetSubRange(i); - Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); + Buffer subBuffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size); subBuffer.SynchronizeMemory(subRange.Address, subRange.Size); @@ -957,7 +941,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (size != 0) { - buffer = _buffers.FindFirstOverlap(address, size); + buffer = _buffers.FindOverlapFast(address, size); buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); @@ -969,7 +953,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - buffer = _buffers.FindFirstOverlap(address, 1); + buffer = _buffers.FindOverlapFast(address, 1); } return buffer; @@ -1007,7 +991,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (size != 0) { - Buffer buffer = _buffers.FindFirstOverlap(address, size); + Buffer buffer = _buffers.FindOverlapFast(address, size); if (copyBackVirtual) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index cb99b455b..351a00e9f 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -761,7 +761,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); + bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; BufferRange range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, bufferStage | BufferStageUtils.FromUsage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, bufferStage); @@ -798,7 +798,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!bounds.IsUnmapped) { - bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); + bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write; BufferRange range = isStorage ? bufferCache.GetBufferRangeAligned(bounds.Range, BufferStageUtils.ComputeStorage(bounds.Flags), isWrite) : bufferCache.GetBufferRange(bounds.Range, BufferStage.Compute); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index bccbdfd31..a31156121 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -2,7 +2,6 @@ using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System; using System.Linq; -using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { @@ -14,12 +13,12 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address of the range in guest memory. /// - public ulong Address { get; } + public ulong Address { get; internal set; } /// /// Size of the range in bytes. /// - public ulong Size { get; } + public ulong Size { get; internal set; } /// /// End address of the range in guest memory. @@ -77,8 +76,6 @@ namespace Ryujinx.Graphics.Gpu.Memory private BufferMigration _source; private BufferModifiedRangeList _migrationTarget; - private readonly Lock _lock = new(); - /// /// Whether the modified range list has any entries or not. /// @@ -86,10 +83,10 @@ namespace Ryujinx.Graphics.Gpu.Memory { get { - lock (_lock) - { - return Count > 0; - } + Lock.EnterReadLock(); + bool result = Count > 0; + Lock.ExitReadLock(); + return result; } } @@ -114,33 +111,39 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Action to perform for each remaining sub-range of the input range public void ExcludeModifiedRegions(ulong address, ulong size, Action action) { - lock (_lock) + // Slices a given region using the modified regions in the list. Calls the action for the new slices. + bool lockOwner = Lock.IsReadLockHeld; + if (!lockOwner) { - // Slices a given region using the modified regions in the list. Calls the action for the new slices. - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + Lock.EnterReadLock(); + } - int count = FindOverlapsNonOverlapping(address, size, ref overlaps); + FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); - for (int i = 0; i < count; i++) + for (int i = 0; i < overlaps.Length; i++) + { + BufferModifiedRange overlap = overlaps[i].Value; + + if (overlap.Address > address) { - BufferModifiedRange overlap = overlaps[i]; - - if (overlap.Address > address) - { - // The start of the remaining region is uncovered by this overlap. Call the action for it. - action(address, overlap.Address - address); - } - - // Remaining region is after this overlap. - size -= overlap.EndAddress - address; - address = overlap.EndAddress; + // The start of the remaining region is uncovered by this overlap. Call the action for it. + action(address, overlap.Address - address); } - if ((long)size > 0) - { - // If there is any region left after removing the overlaps, signal it. - action(address, size); - } + // Remaining region is after this overlap. + size -= overlap.EndAddress - address; + address = overlap.EndAddress; + } + + if (!lockOwner) + { + Lock.ExitReadLock(); + } + + if ((long)size > 0) + { + // If there is any region left after removing the overlaps, signal it. + action(address, size); } } @@ -152,51 +155,102 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size of the modified region in bytes public void SignalModified(ulong address, ulong size) { - // Must lock, as this can affect flushes from the background thread. - lock (_lock) + // We may overlap with some existing modified regions. They must be cut into by the new entry. + Lock.EnterWriteLock(); + OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size, + out ReadOnlySpan> overlaps); + + ulong endAddress = address + size; + ulong syncNumber = _context.SyncNumber; + + if (overlaps.Length == 0) { - // We may overlap with some existing modified regions. They must be cut into by the new entry. - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); + return; + } - int count = FindOverlapsNonOverlapping(address, size, ref overlaps); + BufferModifiedRange buffPost = null; + bool extendsPost = false; + bool extendsPre = false; - ulong endAddress = address + size; - ulong syncNumber = _context.SyncNumber; - - for (int i = 0; i < count; i++) + if (overlaps.Length == 1) + { + if (overlaps[0].Address == address && overlaps[0].EndAddress == endAddress) { - // The overlaps must be removed or split. + overlaps[0].Value.SyncNumber = syncNumber; + overlaps[0].Value.Parent = this; + Lock.ExitWriteLock(); + return; + } - BufferModifiedRange overlap = overlaps[i]; + if (overlaps[0].Address < address) + { + overlaps[0].Value.Size = address - overlaps[0].Address; - if (overlap.Address == address && overlap.Size == size) + extendsPre = true; + + if (overlaps[0].EndAddress > endAddress) { - // Region already exists. Just update the existing sync number. - overlap.SyncNumber = syncNumber; - overlap.Parent = this; - - return; + buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPost = true; } - - Remove(overlap); - - if (overlap.Address < address && overlap.EndAddress > address) + } + else + { + if (overlaps[0].EndAddress > endAddress) { - // A split item must be created behind this overlap. - - Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent)); + overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress; + overlaps[0].Value.Address = endAddress; } - - if (overlap.Address < endAddress && overlap.EndAddress > endAddress) + else { - // A split item must be created after this overlap. - - Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent)); + RemoveAt(result.StartIndex); } } + if (extendsPre && extendsPost) + { + Add(buffPost); + } + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); + + return; } + + BufferModifiedRange buffPre = null; + + if (overlaps[0].Address < address) + { + buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPre = true; + } + + if (overlaps[^1].EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress, + overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent); + extendsPost = true; + } + + RemoveRange(result); + + if (extendsPre) + { + Add(buffPre); + } + + if (extendsPost) + { + Add(buffPost); + } + + Add(new BufferModifiedRange(address, size, syncNumber, this)); + Lock.ExitWriteLock(); } /// @@ -208,25 +262,20 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action rangeAction) { - int count = 0; + Lock.EnterReadLock(); + FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); - - // Range list must be consistent for this operation. - lock (_lock) + for (int i = 0; i < overlaps.Length; i++) { - count = FindOverlapsNonOverlapping(address, size, ref overlaps); - } - - for (int i = 0; i < count; i++) - { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; if (overlap.SyncNumber == syncNumber) { rangeAction(overlap.Address, overlap.Size); } } + + Lock.ExitReadLock(); } /// @@ -237,19 +286,16 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void GetRanges(ulong address, ulong size, Action rangeAction) { - int count = 0; + RangeItem[] overlaps = new RangeItem[1]; + + // We use the non-span method here because keeping the lock will cause a deadlock. + Lock.EnterReadLock(); + OverlapResult result = FindOverlapsNonOverlapping(address, size, ref overlaps); + Lock.ExitReadLock(); - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); - - // Range list must be consistent for this operation. - lock (_lock) + for (int i = 0; i < result.Count; i++) { - count = FindOverlapsNonOverlapping(address, size, ref overlaps); - } - - for (int i = 0; i < count; i++) - { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; rangeAction(overlap.Address, overlap.Size); } } @@ -262,11 +308,10 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if a range exists in the specified region, false otherwise public bool HasRange(ulong address, ulong size) { - // Range list must be consistent for this operation. - lock (_lock) - { - return FindOverlapsNonOverlapping(address, size, ref ThreadStaticArray.Get()) > 0; - } + Lock.EnterReadLock(); + bool result = FindOverlapsNonOverlapping(address, size).Count > 0; + Lock.ExitReadLock(); + return result; } /// @@ -298,38 +343,37 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The start address of the flush range /// The end address of the flush range private void RemoveRangesAndFlush( - BufferModifiedRange[] overlaps, + RangeItem[] overlaps, int rangeCount, long highestDiff, ulong currentSync, ulong address, ulong endAddress) { - lock (_lock) + if (_migrationTarget == null) { - if (_migrationTarget == null) + ulong waitSync = currentSync + (ulong)highestDiff; + + for (int i = 0; i < rangeCount; i++) { - ulong waitSync = currentSync + (ulong)highestDiff; + BufferModifiedRange overlap = overlaps[i].Value; - for (int i = 0; i < rangeCount; i++) + long diff = (long)(overlap.SyncNumber - currentSync); + + if (diff <= highestDiff) { - BufferModifiedRange overlap = overlaps[i]; + ulong clampAddress = Math.Max(address, overlap.Address); + ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); - long diff = (long)(overlap.SyncNumber - currentSync); + Lock.EnterWriteLock(); + ClearPart(overlap, clampAddress, clampEnd); + Lock.ExitWriteLock(); - if (diff <= highestDiff) - { - ulong clampAddress = Math.Max(address, overlap.Address); - ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); - - ClearPart(overlap, clampAddress, clampEnd); - - RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); - } + RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction); } - - return; } + + return; } // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine. @@ -353,30 +397,31 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong endAddress = address + size; ulong currentSync = _context.SyncNumber; - int rangeCount = 0; + int rangeCount; - ref BufferModifiedRange[] overlaps = ref ThreadStaticArray.Get(); + ref RangeItem[] overlaps = ref ThreadStaticArray>.Get(); // Range list must be consistent for this operation - lock (_lock) + Lock.EnterReadLock(); + if (_migrationTarget != null) { - if (_migrationTarget != null) - { - rangeCount = -1; - } - else - { - rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps); - } + rangeCount = -1; } + else + { + // We use the non-span method here because the array is partially modified by the code, which would invalidate a span. + rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps).Count; + } + Lock.ExitReadLock(); if (rangeCount == -1) { - _migrationTarget.WaitForAndFlushRanges(address, size); + _migrationTarget!.WaitForAndFlushRanges(address, size); return; } - else if (rangeCount == 0) + + if (rangeCount == 0) { return; } @@ -388,7 +433,7 @@ namespace Ryujinx.Graphics.Gpu.Memory for (int i = 0; i < rangeCount; i++) { - BufferModifiedRange overlap = overlaps[i]; + BufferModifiedRange overlap = overlaps[i].Value; long diff = (long)(overlap.SyncNumber - currentSync); @@ -419,42 +464,39 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The action to call for each modified range public void InheritRanges(BufferModifiedRangeList ranges, Action registerRangeAction) { - BufferModifiedRange[] inheritRanges; + ranges.Lock.EnterReadLock(); + BufferModifiedRange[] inheritRanges = ranges.ToArray(); + ranges.Lock.ExitReadLock(); - lock (ranges._lock) + // Copy over the migration from the previous range list + + BufferMigration oldMigration = ranges._source; + + BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); + ranges._parent.IncrementReferenceCount(); + + if (_source == null) { - inheritRanges = ranges.ToArray(); + // Create a new migration. + _source = new BufferMigration([span], this, _context.SyncNumber); - lock (_lock) - { - // Copy over the migration from the previous range list - - BufferMigration oldMigration = ranges._source; - - BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration); - ranges._parent.IncrementReferenceCount(); - - if (_source == null) - { - // Create a new migration. - _source = new BufferMigration([span], this, _context.SyncNumber); - - _context.RegisterBufferMigration(_source); - } - else - { - // Extend the migration - _source.AddSpanToEnd(span); - } - - ranges._migrationTarget = this; - - foreach (BufferModifiedRange range in inheritRanges) - { - Add(range); - } - } + _context.RegisterBufferMigration(_source); } + else + { + // Extend the migration + _source.AddSpanToEnd(span); + } + + ranges._migrationTarget = this; + + Lock.EnterWriteLock(); + foreach (BufferModifiedRange range in inheritRanges) + { + Add(range); + } + + Lock.ExitWriteLock(); ulong currentSync = _context.SyncNumber; foreach (BufferModifiedRange range in inheritRanges) @@ -473,18 +515,18 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public void SelfMigration() { - lock (_lock) - { - BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), _parent.GetSnapshotFlushAction(), _source); - BufferMigration migration = new([span], this, _context.SyncNumber); + BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), + _parent.GetSnapshotFlushAction(), _source); + BufferMigration migration = new([span], this, _context.SyncNumber); - // Migration target is used to redirect flush actions to the latest range list, - // so we don't need to set it here. (this range list is still the latest) + // Migration target is used to redirect flush actions to the latest range list, + // so we don't need to set it here. (this range list is still the latest) - _context.RegisterBufferMigration(migration); + _context.RegisterBufferMigration(migration); - _source = migration; - } + Lock.EnterWriteLock(); + _source = migration; + Lock.ExitWriteLock(); } /// @@ -493,13 +535,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The migration to remove public void RemoveMigration(BufferMigration migration) { - lock (_lock) + Lock.EnterWriteLock(); + if (_source == migration) { - if (_source == migration) - { - _source = null; - } + _source = null; } + + Lock.ExitWriteLock(); } private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress) @@ -526,22 +568,85 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size to clear public void Clear(ulong address, ulong size) { - lock (_lock) + ulong endAddress = address + size; + Lock.EnterWriteLock(); + OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan> overlaps); + + if (overlaps.Length == 0) { - // This function can be called from any thread, so it cannot use the arrays for background or foreground. - BufferModifiedRange[] toClear = new BufferModifiedRange[1]; - - int rangeCount = FindOverlapsNonOverlapping(address, size, ref toClear); - - ulong endAddress = address + size; - - for (int i = 0; i < rangeCount; i++) - { - BufferModifiedRange overlap = toClear[i]; - - ClearPart(overlap, address, endAddress); - } + Lock.ExitWriteLock(); + return; } + + BufferModifiedRange buffPost = null; + bool extendsPost = false; + bool extendsPre = false; + + if (overlaps.Length == 1) + { + if (overlaps[0].Address < address) + { + overlaps[0].Value.Size = address - overlaps[0].Address; + extendsPre = true; + + if (overlaps[0].EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPost = true; + } + } + else + { + if (overlaps[^1].EndAddress > endAddress) + { + overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress; + overlaps[0].Value.Address = endAddress; + } + else + { + RemoveAt(result.StartIndex); + } + } + + if (extendsPre && extendsPost) + { + Add(buffPost); + } + + Lock.ExitWriteLock(); + return; + } + + BufferModifiedRange buffPre = null; + + if (overlaps[0].Address < address) + { + buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address, + overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent); + extendsPre = true; + } + + if (overlaps[^1].EndAddress > endAddress) + { + buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress, + overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent); + extendsPost = true; + } + + RemoveRange(result); + + if (extendsPre) + { + Add(buffPre); + } + + if (extendsPost) + { + Add(buffPost); + } + + Lock.ExitWriteLock(); } /// @@ -549,10 +654,9 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public void Clear() { - lock (_lock) - { - Count = 0; - } + Lock.EnterWriteLock(); + Count = 0; + Lock.ExitWriteLock(); } } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index 6efb7f334..95e43e341 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -690,11 +690,8 @@ namespace Ryujinx.Graphics.Gpu.Memory if (_pageTable[l0] == null) { _pageTable[l0] = new ulong[PtLvl1Size]; - - for (ulong index = 0; index < PtLvl1Size; index++) - { - _pageTable[l0][index] = PteUnmapped; - } + + Array.Fill(_pageTable[l0], PteUnmapped); } _pageTable[l0][l1] = pte; diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index ac25b3e5d..38b628a98 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -63,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } private readonly RangeList _virtualRanges; - private VirtualRange[] _virtualRangeOverlaps; + private RangeItem[] _virtualRangeOverlaps; private readonly ConcurrentQueue _deferredUnmaps; private int _hasDeferredUnmaps; @@ -75,7 +75,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { _memoryManager = memoryManager; _virtualRanges = []; - _virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity]; + _virtualRangeOverlaps = new RangeItem[BufferCache.OverlapsBufferInitialCapacity]; _deferredUnmaps = new ConcurrentQueue(); } @@ -106,18 +106,17 @@ namespace Ryujinx.Graphics.Gpu.Memory /// True if the range already existed, false if a new one was created and added public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range) { - VirtualRange[] overlaps = _virtualRangeOverlaps; - int overlapsCount; - + OverlapResult result; + if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0) { while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange)) { - overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps); + result = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size); - for (int index = 0; index < overlapsCount; index++) + if (result.StartIndex >= 0) { - _virtualRanges.Remove(overlaps[index]); + _virtualRanges.RemoveRange(result); } } } @@ -126,27 +125,23 @@ namespace Ryujinx.Graphics.Gpu.Memory ulong originalVa = gpuVa; - overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps); - - if (overlapsCount != 0) + _virtualRanges.Lock.EnterWriteLock(); + result = _virtualRanges.FindOverlapsNonOverlappingAsSpan(gpuVa, size, out ReadOnlySpan> overlaps); + + if (overlaps.Length != 0) { // The virtual range already exists. We just need to check if our range fits inside // the existing one, and if not, we must extend the existing one. ulong endAddress = gpuVa + size; - VirtualRange overlap0 = overlaps[0]; + VirtualRange overlap0 = overlaps[0].Value; if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress) { - for (int index = 0; index < overlapsCount; index++) - { - VirtualRange virtualRange = overlaps[index]; - - gpuVa = Math.Min(gpuVa, virtualRange.Address); - endAddress = Math.Max(endAddress, virtualRange.EndAddress); - - _virtualRanges.Remove(virtualRange); - } + gpuVa = Math.Min(gpuVa, overlaps[0].Address); + endAddress = Math.Max(endAddress, overlaps[^1].EndAddress); + + _virtualRanges.RemoveRange(result); ulong newSize = endAddress - gpuVa; MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize); @@ -170,6 +165,7 @@ namespace Ryujinx.Graphics.Gpu.Memory _virtualRanges.Add(virtualRange); } + _virtualRanges.Lock.ExitWriteLock(); ShrinkOverlapsBufferIfNeeded(); diff --git a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs index 894078aee..da3e7983f 100644 --- a/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs +++ b/src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs @@ -19,16 +19,18 @@ namespace Ryujinx.Memory.Range /// Start address of the search region /// Size of the search region /// Factory for creating new ranges - public void GetOrAddRegions(List list, ulong address, ulong size, Func factory) + public void GetOrAddRegions(out List list, ulong address, ulong size, Func factory) { // (regarding the specific case this generalized function is used for) // A new region may be split into multiple parts if multiple virtual regions have mapped to it. // For instance, while a virtual mapping could cover 0-2 in physical space, the space 0-1 may have already been reserved... // So we need to return both the split 0-1 and 1-2 ranges. - T[] results = new T[1]; - int count = FindOverlapsNonOverlapping(address, size, ref results); - + RangeItem[] results = new RangeItem[1]; + int count = FindOverlapsNonOverlapping(address, size, ref results).Count; + list = new List(count); + Lock.EnterWriteLock(); + if (count == 0) { // The region is fully unmapped. Create and add it to the range list. @@ -43,11 +45,12 @@ namespace Ryujinx.Memory.Range for (int i = 0; i < count; i++) { - T region = results[i]; + T region = results[i].Value; if (count == 1 && region.Address == address && region.Size == size) { // Exact match, no splitting required. list.Add(region); + Lock.ExitWriteLock(); return; } @@ -85,6 +88,8 @@ namespace Ryujinx.Memory.Range Add(fillRegion); } } + + Lock.ExitWriteLock(); } /// diff --git a/src/Ryujinx.Memory/Range/RangeList.cs b/src/Ryujinx.Memory/Range/RangeList.cs index 72cef1de0..d575f4583 100644 --- a/src/Ryujinx.Memory/Range/RangeList.cs +++ b/src/Ryujinx.Memory/Range/RangeList.cs @@ -2,44 +2,63 @@ using System; using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Threading; namespace Ryujinx.Memory.Range { + public readonly struct RangeItem(TValue value) where TValue : IRange + { + public readonly ulong Address = value.Address; + public readonly ulong EndAddress = value.Address + value.Size; + + public readonly TValue Value = value; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool OverlapsWith(ulong address, ulong endAddress) + { + return Address < endAddress && address < EndAddress; + } + } + + /// + /// Result of an Overlaps Finder function. + /// + /// + /// startIndex is inclusive. + /// endIndex is exclusive. + /// + public readonly struct OverlapResult + { + public readonly int StartIndex = -1; + public readonly int EndIndex = -1; + public int Count => EndIndex - StartIndex; + + public OverlapResult(int startIndex, int endIndex) + { + this.StartIndex = startIndex; + this.EndIndex = endIndex; + } + } + /// /// Sorted list of ranges that supports binary search. /// /// Type of the range. public class RangeList : IEnumerable where T : IRange { - private readonly struct RangeItem where TValue : IRange - { - public readonly ulong Address; - public readonly ulong EndAddress; - - public readonly TValue Value; - - public RangeItem(TValue value) - { - Value = value; - - Address = value.Address; - EndAddress = value.Address + value.Size; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool OverlapsWith(ulong address, ulong endAddress) - { - return Address < endAddress && address < EndAddress; - } - } - private const int BackingInitialSize = 1024; - private const int ArrayGrowthSize = 32; private RangeItem[] _items; private readonly int _backingGrowthSize; public int Count { get; protected set; } + + public readonly ReaderWriterLockSlim Lock = new(); + + private const int QuickAccessLength = 8; + private int _offset; + private int _count; + private RangeItem[] _quickAccess = new RangeItem[QuickAccessLength]; /// /// Creates a new range list. @@ -72,22 +91,21 @@ namespace Ryujinx.Memory.Range /// /// The item to be updated /// True if the item was located and updated, false otherwise - public bool Update(T item) + protected bool Update(T item) { int index = BinarySearch(item.Address); if (index >= 0) { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - while (index < Count) { if (_items[index].Value.Equals(item)) { _items[index] = new RangeItem(item); + + _quickAccess = new RangeItem[QuickAccessLength]; + _count = 0; + _offset = 0; return true; } @@ -129,12 +147,34 @@ namespace Ryujinx.Memory.Range } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void RemoveAt(int index) + protected void RemoveAt(int index) { if (index < --Count) { Array.Copy(_items, index + 1, _items, index, Count - index); } + + _quickAccess = new RangeItem[QuickAccessLength]; + _count = 0; + _offset = 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void RemoveRange(OverlapResult overlapResult) + { + if (overlapResult.EndIndex < Count) + { + Array.Copy(_items, overlapResult.EndIndex, _items, overlapResult.StartIndex, Count - overlapResult.EndIndex); + Count -= overlapResult.Count; + } + else if (overlapResult.EndIndex == Count) + { + Count = overlapResult.StartIndex; + } + + _quickAccess = new RangeItem[QuickAccessLength]; + _count = 0; + _offset = 0; } /// @@ -148,11 +188,6 @@ namespace Ryujinx.Memory.Range if (index >= 0) { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - while (index < Count) { if (_items[index].Value.Equals(item)) @@ -175,67 +210,39 @@ namespace Ryujinx.Memory.Range } /// - /// Updates an item's end address. - /// - /// The item to be updated - public void UpdateEndAddress(T item) - { - int index = BinarySearch(item.Address); - - if (index >= 0) - { - while (index > 0 && _items[index - 1].Address == item.Address) - { - index--; - } - - while (index < Count) - { - if (_items[index].Value.Equals(item)) - { - _items[index] = new RangeItem(item); - - return; - } - - if (_items[index].Address > item.Address) - { - break; - } - - index++; - } - } - } - - /// - /// Gets the first item on the list overlapping in memory with the specified item. + /// Gets an item on the list overlapping the specified memory range. /// /// - /// Despite the name, this has no ordering guarantees of the returned item. - /// It only ensures that the item returned overlaps the specified item. - /// - /// Item to check for overlaps - /// The overlapping item, or the default value for the type if none found - public T FindFirstOverlap(T item) - { - return FindFirstOverlap(item.Address, item.Size); - } - - /// - /// Gets the first item on the list overlapping the specified memory range. - /// - /// - /// Despite the name, this has no ordering guarantees of the returned item. + /// This has no ordering guarantees of the returned item. /// It only ensures that the item returned overlaps the specified memory range. /// /// Start address of the range /// Size in bytes of the range /// The overlapping item, or the default value for the type if none found - public T FindFirstOverlap(ulong address, ulong size) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public T FindOverlapFast(ulong address, ulong size) { + for (int i = 0; i < _quickAccess.Length; i++) + { + ref RangeItem item = ref _quickAccess[(i + _offset) % _quickAccess.Length]; + + if (item.OverlapsWith(address, address + size)) + { + return item.Value; + } + } + int index = BinarySearch(address, address + size); + if (_count < _quickAccess.Length) + { + _quickAccess[_count++] = _items[index]; + } + else + { + _quickAccess[_offset++ % _quickAccess.Length] = _items[index]; + } + if (index < 0) { return default; @@ -243,68 +250,66 @@ namespace Ryujinx.Memory.Range return _items[index].Value; } - - /// - /// Gets all items overlapping with the specified item in memory. - /// - /// Item to check for overlaps - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlaps(T item, ref T[] output) - { - return FindOverlaps(item.Address, item.Size, ref output); - } - + /// /// Gets all items on the list overlapping the specified memory range. /// /// Start address of the range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlaps(ulong address, ulong size, ref T[] output) + /// Range information of overlapping items found + public OverlapResult FindOverlaps(ulong address, ulong size, ref RangeItem[] output) { int outputIndex = 0; ulong endAddress = address + size; + + int startIndex = BinarySearch(address); + if (startIndex < 0) + startIndex = ~startIndex; + int endIndex = -1; - for (int i = 0; i < Count; i++) + for (int i = startIndex; i < Count; i++) { ref RangeItem item = ref _items[i]; if (item.Address >= endAddress) { + endIndex = i; break; } if (item.OverlapsWith(address, endAddress)) { - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = item.Value; + outputIndex++; } } - return outputIndex; - } + if (endIndex == -1 && outputIndex > 0) + { + endIndex = Count; + } - /// - /// Gets all items overlapping with the specified item in memory. - /// - /// - /// This method only returns correct results if none of the items on the list overlaps with - /// each other. If that is not the case, this method should not be used. - /// This method is faster than the regular method to find all overlaps. - /// - /// Item to check for overlaps - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlapsNonOverlapping(T item, ref T[] output) - { - return FindOverlapsNonOverlapping(item.Address, item.Size, ref output); + if (outputIndex > 0 && outputIndex == endIndex - startIndex) + { + Array.Resize(ref output, outputIndex); + Array.Copy(_items, endIndex - outputIndex, output, 0, outputIndex); + + return new OverlapResult(startIndex, endIndex); + } + else if (outputIndex > 0) + { + Array.Resize(ref output, outputIndex); + int arrIndex = 0; + for (int i = startIndex; i < endIndex; i++) + { + output[arrIndex++] = _items[i]; + } + + return new OverlapResult(endIndex - outputIndex, endIndex); + } + + return new OverlapResult(); } /// @@ -318,77 +323,68 @@ namespace Ryujinx.Memory.Range /// Start address of the range /// Size in bytes of the range /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of overlapping items found - public int FindOverlapsNonOverlapping(ulong address, ulong size, ref T[] output) + /// Range information of overlapping items found + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size, ref RangeItem[] output) { // This is a bit faster than FindOverlaps, but only works // when none of the items on the list overlaps with each other. - int outputIndex = 0; ulong endAddress = address + size; - int index = BinarySearch(address, endAddress); + (int index, int endIndex) = BinarySearchEdges(address, endAddress); if (index >= 0) { - while (index > 0 && _items[index - 1].OverlapsWith(address, endAddress)) - { - index--; - } - - do - { - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = _items[index++].Value; - } - while (index < Count && _items[index].OverlapsWith(address, endAddress)); + Array.Resize(ref output, endIndex - index); + Array.Copy(_items, index, output, 0, endIndex - index); } - return outputIndex; + return new OverlapResult(index, endIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public OverlapResult FindOverlapsNonOverlappingAsSpan(ulong address, ulong size, out ReadOnlySpan> span) + { + // This is a bit faster than FindOverlaps, but only works + // when none of the items on the list overlaps with each other. + + ulong endAddress = address + size; + + (int index, int endIndex) = BinarySearchEdges(address, endAddress); + + if (index >= 0) + { + span = new ReadOnlySpan>(_items, index, endIndex - index); + return new OverlapResult(index, endIndex); + } + + span = ReadOnlySpan>.Empty; + return new OverlapResult(index, endIndex); } /// - /// Gets all items on the list with the specified memory address. + /// Gets the range of all items on the list overlapping the specified memory range. /// - /// Address to find - /// Output array where matches will be written. It is automatically resized to fit the results - /// The number of matches found - public int FindOverlaps(ulong address, ref T[] output) + /// + /// This method only returns correct results if none of the items on the list overlaps with + /// each other. If that is not the case, this method should not be used. + /// This method is faster than the regular method to find all overlaps. + /// + /// Start address of the range + /// Size in bytes of the range + /// Range information of overlapping items found + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size) { - int index = BinarySearch(address); + // This is a bit faster than FindOverlaps, but only works + // when none of the items on the list overlaps with each other. - int outputIndex = 0; + ulong endAddress = address + size; - if (index >= 0) - { - while (index > 0 && _items[index - 1].Address == address) - { - index--; - } + (int index, int endIndex) = BinarySearchEdges(address, endAddress); - while (index < Count) - { - ref RangeItem overlap = ref _items[index++]; - - if (overlap.Address != address) - { - break; - } - - if (outputIndex == output.Length) - { - Array.Resize(ref output, outputIndex + ArrayGrowthSize); - } - - output[outputIndex++] = overlap.Value; - } - } - - return outputIndex; + return new OverlapResult(index, endIndex); } /// @@ -396,6 +392,7 @@ namespace Ryujinx.Memory.Range /// /// Address to find /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int BinarySearch(ulong address) { int left = 0; @@ -426,13 +423,14 @@ namespace Ryujinx.Memory.Range return ~left; } - + /// /// Performs binary search for items overlapping a given memory range. /// /// Start address of the range /// End address of the range /// List index of the item, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int BinarySearch(ulong address, ulong endAddress) { int left = 0; @@ -464,6 +462,136 @@ namespace Ryujinx.Memory.Range return ~left; } + /// + /// Performs binary search for items overlapping a given memory range. + /// + /// Start address of the range + /// End address of the range + /// Range information (inclusive, exclusive) of items that overlaps, or complement index of nearest item with lower value on the list + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private (int, int) BinarySearchEdges(ulong address, ulong endAddress) + { + if (Count == 0) + return (~0, ~0); + + if (Count == 1) + { + ref RangeItem item = ref _items[0]; + + if (item.OverlapsWith(address, endAddress)) + { + return (0, 1); + } + + if (address < item.Address) + { + return (~0, ~0); + } + else + { + return (~1, ~1); + } + } + + int left = 0; + int right = Count - 1; + + int leftEdge = -1; + int rightEdgeMatch = -1; + int rightEdgeNoMatch = -1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + { + leftEdge = middle; + break; + } + else if (address < item.Address) + { + return (~right, ~right); + } + else + { + return (~(right + 1), ~(right + 1)); + } + } + + if (match) + { + right = middle; + if (rightEdgeMatch == -1) + rightEdgeMatch = middle; + } + else if (address < item.Address) + { + right = middle - 1; + rightEdgeNoMatch = middle; + } + else + { + left = middle + 1; + } + } + + if (left > right) + { + return (~left, ~left); + } + + if (rightEdgeMatch == -1) + { + return (leftEdge, leftEdge + 1); + } + + left = rightEdgeMatch; + right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = right - (range >> 1); + + ref RangeItem item = ref _items[middle]; + + bool match = item.OverlapsWith(address, endAddress); + + if (range == 0) + { + if (match) + return (leftEdge, middle + 1); + else + return (leftEdge, middle); + } + + if (match) + { + left = middle; + } + else if (address < item.Address) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return (leftEdge, right + 1); + } + public IEnumerator GetEnumerator() { for (int i = 0; i < Count; i++) diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index e7791fec3..7aecd7f4d 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -1,5 +1,6 @@ using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; +using System; using System.Collections.Generic; namespace Ryujinx.Memory.Tracking @@ -76,17 +77,15 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - - int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + regions.Lock.EnterReadLock(); + regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan> overlaps); + + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = overlaps[i].Value; // If the region has been fully remapped, signal that it has been mapped again. bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); @@ -97,6 +96,7 @@ namespace Ryujinx.Memory.Tracking region.UpdateProtection(); } + regions.Lock.ExitReadLock(); } } } @@ -114,20 +114,19 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; + regions.Lock.EnterReadLock(); + regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan> overlaps); - int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + for (int i = 0; i < overlaps.Length; i++) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = overlaps[i].Value; region.SignalMappingChanged(false); } + regions.Lock.ExitReadLock(); } } } @@ -165,10 +164,11 @@ namespace Ryujinx.Memory.Tracking /// A list of virtual regions within the given range internal List GetVirtualRegionsForHandle(ulong va, ulong size, bool guest) { - List result = []; NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; - regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); - + regions.Lock.EnterUpgradeableReadLock(); + regions.GetOrAddRegions(out List result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); + regions.Lock.ExitUpgradeableReadLock(); + return result; } @@ -296,25 +296,26 @@ namespace Ryujinx.Memory.Tracking lock (TrackingLock) { - ref VirtualRegion[] overlaps = ref ThreadStaticArray.Get(); - NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; + ref RangeItem[] overlaps = ref ThreadStaticArray>.Get(); + + // We use the non-span method here because keeping the lock will cause a deadlock. + regions.Lock.EnterReadLock(); + OverlapResult result = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); + regions.Lock.ExitReadLock(); - int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); - - if (count == 0 && !precise) + if (overlaps.Length == 0 && !precise) { if (_memoryManager.IsRangeMapped(address, size)) { // TODO: There is currently the possibility that a page can be protected after its virtual region is removed. // This code handles that case when it happens, but it would be better to find out how this happens. _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest); + return true; // This memory _should_ be mapped, so we need to try again. } - else - { - shouldThrow = true; - } + + shouldThrow = true; } else { @@ -324,9 +325,9 @@ namespace Ryujinx.Memory.Tracking size += (ulong)_pageSize; } - for (int i = 0; i < count; i++) + for (int i = 0; i < result.Count; i++) { - VirtualRegion region = overlaps[i]; + VirtualRegion region = overlaps[i].Value; if (precise) {