crafterpika/ryujinx
1
0
Fork 0
mirror of https://git.ryujinx.app/ryubing/ryujinx.git synced 2025-06-28 00:16:23 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'memory-refactor' into 'master'

Browse source 
Memory RangeList Refactor

See merge request [ryubing/ryujinx!46](https://git.ryujinx.app/ryubing/ryujinx/-/merge_requests/46)
This commit is contained in:
LotP 2025-06-23 19:37:10 -05:00
commit 2f821e5b2f
10 changed files with 723 additions and 506 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View file

@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong size, ulong size,
BufferStage stage, BufferStage stage,
bool sparseCompatible, bool sparseCompatible,
IEnumerable<Buffer> baseBuffers = null) ReadOnlySpan<RangeItem<Buffer>> baseBuffers)
{ {
_context = context; _context = context;
_physicalMemory = physicalMemory; _physicalMemory = physicalMemory;
@ -126,21 +126,22 @@ namespace Ryujinx.Graphics.Gpu.Memory
_useGranular = size > GranularBufferThreshold; _useGranular = size > GranularBufferThreshold;
IEnumerable<IRegionHandle> baseHandles = null; List<IRegionHandle> baseHandles = null;
if (baseBuffers != null) if (!baseBuffers.IsEmpty)
{ {
baseHandles = baseBuffers.SelectMany(buffer => baseHandles = new List<IRegionHandle>();
foreach (RangeItem<Buffer> buffer in baseBuffers)
{ {
if (buffer._useGranular) if (buffer.Value._useGranular)
{ {
return buffer._memoryTrackingGranular.GetHandles(); baseHandles.AddRange((buffer.Value._memoryTrackingGranular.GetHandles()));
} }
else else
{ {
return Enumerable.Repeat(buffer._memoryTracking, 1); baseHandles.Add(buffer.Value._memoryTracking);
} }
}); }
} }
if (_useGranular) if (_useGranular)

11
src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="parent">Parent buffer</param> /// <param name="parent">Parent buffer</param>
/// <param name="stage">Initial buffer stage</param> /// <param name="stage">Initial buffer stage</param>
/// <param name="baseBuffers">Buffers to inherit state from</param> /// <param name="baseBuffers">Buffers to inherit state from</param>
public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable<Buffer> baseBuffers = null) public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, ReadOnlySpan<RangeItem<Buffer>> baseBuffers)
{ {
_size = (int)parent.Size; _size = (int)parent.Size;
_systemMemoryType = context.Capabilities.MemoryType; _systemMemoryType = context.Capabilities.MemoryType;
@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferStage storageFlags = stage & BufferStage.StorageMask; BufferStage storageFlags = stage & BufferStage.StorageMask;
if (parent.Size > DeviceLocalSizeThreshold && baseBuffers == null) if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.IsEmpty)
{ {
_desiredType = BufferBackingType.DeviceMemory; _desiredType = BufferBackingType.DeviceMemory;
} }
@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// TODO: Might be nice to force atomic access to be device local for any stage. // TODO: Might be nice to force atomic access to be device local for any stage.
} }
if (baseBuffers != null) if (!baseBuffers.IsEmpty)
{ {
foreach (Buffer buffer in baseBuffers) foreach (RangeItem<Buffer> buffer in baseBuffers)
{ {
CombineState(buffer.BackingState); CombineState(buffer.Value.BackingState);
} }
} }
} }

134
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
View file

@ -2,7 +2,6 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range; using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
@ -42,7 +41,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
private readonly RangeList<Buffer> _buffers; private readonly RangeList<Buffer> _buffers;
private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers; private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers;
private Buffer[] _bufferOverlaps; private RangeItem<Buffer>[] _bufferOverlaps;
private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache; private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache;
private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache; private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache;
@ -64,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_buffers = []; _buffers = [];
_multiRangeBuffers = []; _multiRangeBuffers = [];
_bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity]; _bufferOverlaps = new RangeItem<Buffer>[OverlapsBufferInitialCapacity];
_dirtyCache = new Dictionary<ulong, BufferCacheEntry>(); _dirtyCache = new Dictionary<ulong, BufferCacheEntry>();
@ -79,7 +78,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="e">Event arguments</param> /// <param name="e">Event arguments</param>
public void MemoryUnmappedHandler(object sender, UnmapEventArgs e) public void MemoryUnmappedHandler(object sender, UnmapEventArgs e)
{ {
Buffer[] overlaps = new Buffer[10]; RangeItem<Buffer>[] overlaps = new RangeItem<Buffer>[10];
int overlapCount; int overlapCount;
MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size); MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
@ -90,12 +89,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
lock (_buffers) lock (_buffers)
{ {
overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps); overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps).Count;
} }
for (int i = 0; i < overlapCount; i++) for (int i = 0; i < overlapCount; i++)
{ {
overlaps[i].Unmapped(subRange.Address, subRange.Size); overlaps[i].Value.Unmapped(subRange.Address, subRange.Size);
} }
} }
} }
@ -336,7 +335,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
ulong alignedSize = alignedEndAddress - alignedAddress; ulong alignedSize = alignedEndAddress - alignedAddress;
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); Buffer buffer = _buffers.FindOverlapFast(alignedAddress, alignedSize);
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
@ -403,7 +402,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (subRange.Address != MemoryManager.PteUnmapped) if (subRange.Address != MemoryManager.PteUnmapped)
{ {
Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); Buffer buffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size);
virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size);
physicalBuffers.Add(buffer); physicalBuffers.Add(buffer);
@ -495,10 +494,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="stage">The type of usage that created the buffer</param> /// <param name="stage">The type of usage that created the buffer</param>
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage) private void CreateBufferAligned(ulong address, ulong size, BufferStage stage)
{ {
Buffer[] overlaps = _bufferOverlaps; _buffers.Lock.EnterWriteLock();
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<Buffer>> overlaps);
if (overlapsCount != 0) if (result.Count != 0)
{ {
// The buffer already exists. We can just return the existing buffer // The buffer already exists. We can just return the existing buffer
// if the buffer we need is fully contained inside the overlapping buffer. // if the buffer we need is fully contained inside the overlapping buffer.
@ -507,7 +506,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
// old buffer(s) to the new buffer. // old buffer(s) to the new buffer.
ulong endAddress = address + size; ulong endAddress = address + size;
Buffer overlap0 = overlaps[0]; RangeItem<Buffer> overlap0 = overlaps[0];
if (overlap0.Address > address || overlap0.EndAddress < endAddress) if (overlap0.Address > address || overlap0.EndAddress < endAddress)
{ {
@ -522,51 +521,51 @@ namespace Ryujinx.Graphics.Gpu.Memory
// sequential memory. // sequential memory.
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the // Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages. // range crosses a page, and after alignment, ends having a size of 2 pages.
if (overlapsCount == 1 && if (result.Count == 1 &&
address >= overlap0.Address && address >= overlap0.Address &&
endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2) endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2)
{ {
// Try to grow the buffer by 1.5x of its current size. // Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts. // This improves performance in the cases where the buffer is resized often by small amounts.
ulong existingSize = overlap0.Size; ulong existingSize = overlap0.Value.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize); size = Math.Max(size, growthSize);
endAddress = address + size; endAddress = address + size;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out overlaps);
} }
address = Math.Min(address, overlaps[0].Address);
endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
for (int index = 0; index < overlapsCount; index++) for (int index = 0; index < result.Count; index++)
{ {
Buffer buffer = overlaps[index]; RangeItem<Buffer> buffer = overlaps[index];
anySparseCompatible |= buffer.SparseCompatible; anySparseCompatible |= buffer.Value.SparseCompatible;
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
lock (_buffers)
{
_buffers.Remove(buffer);
}
} }
ulong newSize = endAddress - address; ulong newSize = endAddress - address;
CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, overlapsCount); Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps);
_buffers.RemoveRange(result);
_buffers.Add(newBuffer);
} }
} }
else else
{ {
// No overlap, just create a new buffer. // No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false); Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false, ReadOnlySpan<RangeItem<Buffer>>.Empty);
lock (_buffers) _buffers.Add(buffer);
{
_buffers.Add(buffer);
}
} }
_buffers.Lock.ExitWriteLock();
ShrinkOverlapsBufferIfNeeded(); ShrinkOverlapsBufferIfNeeded();
} }
@ -582,22 +581,22 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="alignment">Alignment of the start address of the buffer</param> /// <param name="alignment">Alignment of the start address of the buffer</param>
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment) private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment)
{ {
Buffer[] overlaps = _bufferOverlaps; _buffers.Lock.EnterWriteLock();
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps); OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<Buffer>> overlaps);
bool sparseAligned = alignment >= SparseBufferAlignmentSize; bool sparseAligned = alignment >= SparseBufferAlignmentSize;
if (overlapsCount != 0) if (result.Count != 0)
{ {
// If the buffer already exists, make sure if covers the entire range, // If the buffer already exists, make sure if covers the entire range,
// and make sure it is properly aligned, otherwise sparse mapping may fail. // and make sure it is properly aligned, otherwise sparse mapping may fail.
ulong endAddress = address + size; ulong endAddress = address + size;
Buffer overlap0 = overlaps[0]; RangeItem<Buffer> overlap0 = overlaps[0];
if (overlap0.Address > address || if (overlap0.Address > address ||
overlap0.EndAddress < endAddress || overlap0.EndAddress < endAddress ||
(overlap0.Address & (alignment - 1)) != 0 || (overlap0.Address & (alignment - 1)) != 0 ||
(!overlap0.SparseCompatible && sparseAligned)) (!overlap0.Value.SparseCompatible && sparseAligned))
{ {
// We need to make sure the new buffer is properly aligned. // We need to make sure the new buffer is properly aligned.
// However, after the range is aligned, it is possible that it // However, after the range is aligned, it is possible that it
@ -605,47 +604,36 @@ namespace Ryujinx.Graphics.Gpu.Memory
// and ensure we cover all overlaps. // and ensure we cover all overlaps.
int oldOverlapsCount; int oldOverlapsCount;
endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
do do
{ {
for (int index = 0; index < overlapsCount; index++) address = Math.Min(address, overlaps[0].Address);
{
Buffer buffer = overlaps[index];
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
}
address &= ~(alignment - 1); address &= ~(alignment - 1);
oldOverlapsCount = overlapsCount; oldOverlapsCount = result.Count;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps); result = _buffers.FindOverlapsNonOverlappingAsSpan(address, endAddress - address, out overlaps);
}
while (oldOverlapsCount != overlapsCount);
lock (_buffers)
{
for (int index = 0; index < overlapsCount; index++)
{
_buffers.Remove(overlaps[index]);
}
} }
while (oldOverlapsCount != result.Count);
ulong newSize = endAddress - address; ulong newSize = endAddress - address;
CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, overlapsCount); Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps);
_buffers.RemoveRange(result);
_buffers.Add(newBuffer);
} }
} }
else else
{ {
// No overlap, just create a new buffer. // No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned); Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned, ReadOnlySpan<RangeItem<Buffer>>.Empty);
lock (_buffers) _buffers.Add(buffer);
{
_buffers.Add(buffer);
}
} }
_buffers.Lock.ExitWriteLock();
ShrinkOverlapsBufferIfNeeded(); ShrinkOverlapsBufferIfNeeded();
} }
@ -660,19 +648,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="stage">The type of usage that created the buffer</param> /// <param name="stage">The type of usage that created the buffer</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param> /// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="overlaps">Buffers overlapping the range</param> /// <param name="overlaps">Buffers overlapping the range</param>
/// <param name="overlapsCount">Total of overlaps</param> private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, ReadOnlySpan<RangeItem<Buffer>> overlaps)
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, Buffer[] overlaps, int overlapsCount)
{ {
Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps.Take(overlapsCount)); Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps);
lock (_buffers) for (int index = 0; index < overlaps.Length; index++)
{ {
_buffers.Add(newBuffer); Buffer buffer = overlaps[index].Value;
}
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = overlaps[index];
int dstOffset = (int)(buffer.Address - newBuffer.Address); int dstOffset = (int)(buffer.Address - newBuffer.Address);
@ -688,6 +670,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
NotifyBuffersModified?.Invoke(); NotifyBuffersModified?.Invoke();
RecreateMultiRangeBuffers(address, size); RecreateMultiRangeBuffers(address, size);
return newBuffer;
} }
/// <summary> /// <summary>
@ -909,7 +893,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
MemoryRange subRange = range.GetSubRange(i); MemoryRange subRange = range.GetSubRange(i);
Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); Buffer subBuffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size);
subBuffer.SynchronizeMemory(subRange.Address, subRange.Size); subBuffer.SynchronizeMemory(subRange.Address, subRange.Size);
@ -957,7 +941,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (size != 0) if (size != 0)
{ {
buffer = _buffers.FindFirstOverlap(address, size); buffer = _buffers.FindOverlapFast(address, size);
buffer.CopyFromDependantVirtualBuffers(); buffer.CopyFromDependantVirtualBuffers();
buffer.SynchronizeMemory(address, size); buffer.SynchronizeMemory(address, size);
@ -969,7 +953,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
} }
else else
{ {
buffer = _buffers.FindFirstOverlap(address, 1); buffer = _buffers.FindOverlapFast(address, 1);
} }
return buffer; return buffer;
@ -1007,7 +991,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
if (size != 0) if (size != 0)
{ {
Buffer buffer = _buffers.FindFirstOverlap(address, size); Buffer buffer = _buffers.FindOverlapFast(address, size);
if (copyBackVirtual) if (copyBackVirtual)
{ {

4
src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
View file

@ -761,7 +761,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (!bounds.IsUnmapped) if (!bounds.IsUnmapped)
{ {
bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write;
BufferRange range = isStorage BufferRange range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Range, bufferStage | BufferStageUtils.FromUsage(bounds.Flags), isWrite) ? bufferCache.GetBufferRangeAligned(bounds.Range, bufferStage | BufferStageUtils.FromUsage(bounds.Flags), isWrite)
: bufferCache.GetBufferRange(bounds.Range, bufferStage); : bufferCache.GetBufferRange(bounds.Range, bufferStage);
@ -798,7 +798,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (!bounds.IsUnmapped) if (!bounds.IsUnmapped)
{ {
bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write;
BufferRange range = isStorage BufferRange range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Range, BufferStageUtils.ComputeStorage(bounds.Flags), isWrite) ? bufferCache.GetBufferRangeAligned(bounds.Range, BufferStageUtils.ComputeStorage(bounds.Flags), isWrite)
: bufferCache.GetBufferRange(bounds.Range, BufferStage.Compute); : bufferCache.GetBufferRange(bounds.Range, BufferStage.Compute);

468
src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
View file

@ -2,7 +2,6 @@ using Ryujinx.Common.Pools;
using Ryujinx.Memory.Range; using Ryujinx.Memory.Range;
using System; using System;
using System.Linq; using System.Linq;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
@ -14,12 +13,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <summary> /// <summary>
/// Start address of the range in guest memory. /// Start address of the range in guest memory.
/// </summary> /// </summary>
public ulong Address { get; } public ulong Address { get; internal set; }
/// <summary> /// <summary>
/// Size of the range in bytes. /// Size of the range in bytes.
/// </summary> /// </summary>
public ulong Size { get; } public ulong Size { get; internal set; }
/// <summary> /// <summary>
/// End address of the range in guest memory. /// End address of the range in guest memory.
@ -77,8 +76,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
private BufferMigration _source; private BufferMigration _source;
private BufferModifiedRangeList _migrationTarget; private BufferModifiedRangeList _migrationTarget;
private readonly Lock _lock = new();
/// <summary> /// <summary>
/// Whether the modified range list has any entries or not. /// Whether the modified range list has any entries or not.
/// </summary> /// </summary>
@ -86,10 +83,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
get get
{ {
lock (_lock) Lock.EnterReadLock();
{ bool result = Count > 0;
return Count > 0; Lock.ExitReadLock();
} return result;
} }
} }
@ -114,33 +111,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="action">Action to perform for each remaining sub-range of the input range</param> /// <param name="action">Action to perform for each remaining sub-range of the input range</param>
public void ExcludeModifiedRegions(ulong address, ulong size, Action<ulong, ulong> action) public void ExcludeModifiedRegions(ulong address, ulong size, Action<ulong, ulong> action)
{ {
lock (_lock) // Slices a given region using the modified regions in the list. Calls the action for the new slices.
bool lockOwner = Lock.IsReadLockHeld;
if (!lockOwner)
{ {
// Slices a given region using the modified regions in the list. Calls the action for the new slices. Lock.EnterReadLock();
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get(); }
int count = FindOverlapsNonOverlapping(address, size, ref overlaps); FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
for (int i = 0; i < count; i++) for (int i = 0; i < overlaps.Length; i++)
{
BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.Address > address)
{ {
BufferModifiedRange overlap = overlaps[i]; // The start of the remaining region is uncovered by this overlap. Call the action for it.
action(address, overlap.Address - address);
if (overlap.Address > address)
{
// The start of the remaining region is uncovered by this overlap. Call the action for it.
action(address, overlap.Address - address);
}
// Remaining region is after this overlap.
size -= overlap.EndAddress - address;
address = overlap.EndAddress;
} }
if ((long)size > 0) // Remaining region is after this overlap.
{ size -= overlap.EndAddress - address;
// If there is any region left after removing the overlaps, signal it. address = overlap.EndAddress;
action(address, size); }
}
if (!lockOwner)
{
Lock.ExitReadLock();
}
if ((long)size > 0)
{
// If there is any region left after removing the overlaps, signal it.
action(address, size);
} }
} }
@ -152,51 +155,102 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size of the modified region in bytes</param> /// <param name="size">Size of the modified region in bytes</param>
public void SignalModified(ulong address, ulong size) public void SignalModified(ulong address, ulong size)
{ {
// Must lock, as this can affect flushes from the background thread. // We may overlap with some existing modified regions. They must be cut into by the new entry.
lock (_lock) Lock.EnterWriteLock();
OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size,
out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
ulong endAddress = address + size;
ulong syncNumber = _context.SyncNumber;
if (overlaps.Length == 0)
{ {
// We may overlap with some existing modified regions. They must be cut into by the new entry. Add(new BufferModifiedRange(address, size, syncNumber, this));
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get(); Lock.ExitWriteLock();
return;
}
int count = FindOverlapsNonOverlapping(address, size, ref overlaps); BufferModifiedRange buffPost = null;
bool extendsPost = false;
bool extendsPre = false;
ulong endAddress = address + size; if (overlaps.Length == 1)
ulong syncNumber = _context.SyncNumber; {
if (overlaps[0].Address == address && overlaps[0].EndAddress == endAddress)
for (int i = 0; i < count; i++)
{ {
// The overlaps must be removed or split. overlaps[0].Value.SyncNumber = syncNumber;
overlaps[0].Value.Parent = this;
Lock.ExitWriteLock();
return;
}
BufferModifiedRange overlap = overlaps[i]; if (overlaps[0].Address < address)
{
overlaps[0].Value.Size = address - overlaps[0].Address;
if (overlap.Address == address && overlap.Size == size) extendsPre = true;
if (overlaps[0].EndAddress > endAddress)
{ {
// Region already exists. Just update the existing sync number. buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress,
overlap.SyncNumber = syncNumber; overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
overlap.Parent = this; extendsPost = true;
return;
} }
}
Remove(overlap); else
{
if (overlap.Address < address && overlap.EndAddress > address) if (overlaps[0].EndAddress > endAddress)
{ {
// A split item must be created behind this overlap. overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress;
overlaps[0].Value.Address = endAddress;
Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent));
} }
else
if (overlap.Address < endAddress && overlap.EndAddress > endAddress)
{ {
// A split item must be created after this overlap. RemoveAt(result.StartIndex);
Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent));
} }
} }
if (extendsPre && extendsPost)
{
Add(buffPost);
}
Add(new BufferModifiedRange(address, size, syncNumber, this)); Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
return;
} }
BufferModifiedRange buffPre = null;
if (overlaps[0].Address < address)
{
buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address,
overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
extendsPre = true;
}
if (overlaps[^1].EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress,
overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent);
extendsPost = true;
}
RemoveRange(result);
if (extendsPre)
{
Add(buffPre);
}
if (extendsPost)
{
Add(buffPost);
}
Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
} }
/// <summary> /// <summary>
@ -208,25 +262,20 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="rangeAction">The action to call for each modified range</param> /// <param name="rangeAction">The action to call for each modified range</param>
public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction) public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction)
{ {
int count = 0; Lock.EnterReadLock();
FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get(); for (int i = 0; i < overlaps.Length; i++)
// Range list must be consistent for this operation.
lock (_lock)
{ {
count = FindOverlapsNonOverlapping(address, size, ref overlaps); BufferModifiedRange overlap = overlaps[i].Value;
}
for (int i = 0; i < count; i++)
{
BufferModifiedRange overlap = overlaps[i];
if (overlap.SyncNumber == syncNumber) if (overlap.SyncNumber == syncNumber)
{ {
rangeAction(overlap.Address, overlap.Size); rangeAction(overlap.Address, overlap.Size);
} }
} }
Lock.ExitReadLock();
} }
/// <summary> /// <summary>
@ -237,19 +286,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="rangeAction">The action to call for each modified range</param> /// <param name="rangeAction">The action to call for each modified range</param>
public void GetRanges(ulong address, ulong size, Action<ulong, ulong> rangeAction) public void GetRanges(ulong address, ulong size, Action<ulong, ulong> rangeAction)
{ {
int count = 0; RangeItem<BufferModifiedRange>[] overlaps = new RangeItem<BufferModifiedRange>[1];
// We use the non-span method here because keeping the lock will cause a deadlock.
Lock.EnterReadLock();
OverlapResult result = FindOverlapsNonOverlapping(address, size, ref overlaps);
Lock.ExitReadLock();
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get(); for (int i = 0; i < result.Count; i++)
// Range list must be consistent for this operation.
lock (_lock)
{ {
count = FindOverlapsNonOverlapping(address, size, ref overlaps); BufferModifiedRange overlap = overlaps[i].Value;
}
for (int i = 0; i < count; i++)
{
BufferModifiedRange overlap = overlaps[i];
rangeAction(overlap.Address, overlap.Size); rangeAction(overlap.Address, overlap.Size);
} }
} }
@ -262,11 +308,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>True if a range exists in the specified region, false otherwise</returns> /// <returns>True if a range exists in the specified region, false otherwise</returns>
public bool HasRange(ulong address, ulong size) public bool HasRange(ulong address, ulong size)
{ {
// Range list must be consistent for this operation. Lock.EnterReadLock();
lock (_lock) bool result = FindOverlapsNonOverlapping(address, size).Count > 0;
{ Lock.ExitReadLock();
return FindOverlapsNonOverlapping(address, size, ref ThreadStaticArray<BufferModifiedRange>.Get()) > 0; return result;
}
} }
/// <summary> /// <summary>
@ -298,38 +343,37 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="address">The start address of the flush range</param> /// <param name="address">The start address of the flush range</param>
/// <param name="endAddress">The end address of the flush range</param> /// <param name="endAddress">The end address of the flush range</param>
private void RemoveRangesAndFlush( private void RemoveRangesAndFlush(
BufferModifiedRange[] overlaps, RangeItem<BufferModifiedRange>[] overlaps,
int rangeCount, int rangeCount,
long highestDiff, long highestDiff,
ulong currentSync, ulong currentSync,
ulong address, ulong address,
ulong endAddress) ulong endAddress)
{ {
lock (_lock) if (_migrationTarget == null)
{ {
if (_migrationTarget == null) ulong waitSync = currentSync + (ulong)highestDiff;
for (int i = 0; i < rangeCount; i++)
{ {
ulong waitSync = currentSync + (ulong)highestDiff; BufferModifiedRange overlap = overlaps[i].Value;
for (int i = 0; i < rangeCount; i++) long diff = (long)(overlap.SyncNumber - currentSync);
if (diff <= highestDiff)
{ {
BufferModifiedRange overlap = overlaps[i]; ulong clampAddress = Math.Max(address, overlap.Address);
ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
long diff = (long)(overlap.SyncNumber - currentSync); Lock.EnterWriteLock();
ClearPart(overlap, clampAddress, clampEnd);
Lock.ExitWriteLock();
if (diff <= highestDiff) RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction);
{
ulong clampAddress = Math.Max(address, overlap.Address);
ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
ClearPart(overlap, clampAddress, clampEnd);
RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction);
}
} }
return;
} }
return;
} }
// There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine. // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine.
@ -353,30 +397,31 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size; ulong endAddress = address + size;
ulong currentSync = _context.SyncNumber; ulong currentSync = _context.SyncNumber;
int rangeCount = 0; int rangeCount;
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get(); ref RangeItem<BufferModifiedRange>[] overlaps = ref ThreadStaticArray<RangeItem<BufferModifiedRange>>.Get();
// Range list must be consistent for this operation // Range list must be consistent for this operation
lock (_lock) Lock.EnterReadLock();
if (_migrationTarget != null)
{ {
if (_migrationTarget != null) rangeCount = -1;
{
rangeCount = -1;
}
else
{
rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps);
}
} }
else
{
// We use the non-span method here because the array is partially modified by the code, which would invalidate a span.
rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps).Count;
}
Lock.ExitReadLock();
if (rangeCount == -1) if (rangeCount == -1)
{ {
_migrationTarget.WaitForAndFlushRanges(address, size); _migrationTarget!.WaitForAndFlushRanges(address, size);
return; return;
} }
else if (rangeCount == 0)
if (rangeCount == 0)
{ {
return; return;
} }
@ -388,7 +433,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < rangeCount; i++) for (int i = 0; i < rangeCount; i++)
{ {
BufferModifiedRange overlap = overlaps[i]; BufferModifiedRange overlap = overlaps[i].Value;
long diff = (long)(overlap.SyncNumber - currentSync); long diff = (long)(overlap.SyncNumber - currentSync);
@ -419,42 +464,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="registerRangeAction">The action to call for each modified range</param> /// <param name="registerRangeAction">The action to call for each modified range</param>
public void InheritRanges(BufferModifiedRangeList ranges, Action<ulong, ulong> registerRangeAction) public void InheritRanges(BufferModifiedRangeList ranges, Action<ulong, ulong> registerRangeAction)
{ {
BufferModifiedRange[] inheritRanges; ranges.Lock.EnterReadLock();
BufferModifiedRange[] inheritRanges = ranges.ToArray();
ranges.Lock.ExitReadLock();
lock (ranges._lock) // Copy over the migration from the previous range list
BufferMigration oldMigration = ranges._source;
BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration);
ranges._parent.IncrementReferenceCount();
if (_source == null)
{ {
inheritRanges = ranges.ToArray(); // Create a new migration.
_source = new BufferMigration([span], this, _context.SyncNumber);
lock (_lock) _context.RegisterBufferMigration(_source);
{
// Copy over the migration from the previous range list
BufferMigration oldMigration = ranges._source;
BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration);
ranges._parent.IncrementReferenceCount();
if (_source == null)
{
// Create a new migration.
_source = new BufferMigration([span], this, _context.SyncNumber);
_context.RegisterBufferMigration(_source);
}
else
{
// Extend the migration
_source.AddSpanToEnd(span);
}
ranges._migrationTarget = this;
foreach (BufferModifiedRange range in inheritRanges)
{
Add(range);
}
}
} }
else
{
// Extend the migration
_source.AddSpanToEnd(span);
}
ranges._migrationTarget = this;
Lock.EnterWriteLock();
foreach (BufferModifiedRange range in inheritRanges)
{
Add(range);
}
Lock.ExitWriteLock();
ulong currentSync = _context.SyncNumber; ulong currentSync = _context.SyncNumber;
foreach (BufferModifiedRange range in inheritRanges) foreach (BufferModifiedRange range in inheritRanges)
@ -473,18 +515,18 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
public void SelfMigration() public void SelfMigration()
{ {
lock (_lock) BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(),
{ _parent.GetSnapshotFlushAction(), _source);
BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), _parent.GetSnapshotFlushAction(), _source); BufferMigration migration = new([span], this, _context.SyncNumber);
BufferMigration migration = new([span], this, _context.SyncNumber);
// Migration target is used to redirect flush actions to the latest range list, // Migration target is used to redirect flush actions to the latest range list,
// so we don't need to set it here. (this range list is still the latest) // so we don't need to set it here. (this range list is still the latest)
_context.RegisterBufferMigration(migration); _context.RegisterBufferMigration(migration);
_source = migration; Lock.EnterWriteLock();
} _source = migration;
Lock.ExitWriteLock();
} }
/// <summary> /// <summary>
@ -493,13 +535,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="migration">The migration to remove</param> /// <param name="migration">The migration to remove</param>
public void RemoveMigration(BufferMigration migration) public void RemoveMigration(BufferMigration migration)
{ {
lock (_lock) Lock.EnterWriteLock();
if (_source == migration)
{ {
if (_source == migration) _source = null;
{
_source = null;
}
} }
Lock.ExitWriteLock();
} }
private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress) private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress)
@ -526,22 +568,85 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size to clear</param> /// <param name="size">Size to clear</param>
public void Clear(ulong address, ulong size) public void Clear(ulong address, ulong size)
{ {
lock (_lock) ulong endAddress = address + size;
Lock.EnterWriteLock();
OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
if (overlaps.Length == 0)
{ {
// This function can be called from any thread, so it cannot use the arrays for background or foreground. Lock.ExitWriteLock();
BufferModifiedRange[] toClear = new BufferModifiedRange[1]; return;
int rangeCount = FindOverlapsNonOverlapping(address, size, ref toClear);
ulong endAddress = address + size;
for (int i = 0; i < rangeCount; i++)
{
BufferModifiedRange overlap = toClear[i];
ClearPart(overlap, address, endAddress);
}
} }
BufferModifiedRange buffPost = null;
bool extendsPost = false;
bool extendsPre = false;
if (overlaps.Length == 1)
{
if (overlaps[0].Address < address)
{
overlaps[0].Value.Size = address - overlaps[0].Address;
extendsPre = true;
if (overlaps[0].EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress,
overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
extendsPost = true;
}
}
else
{
if (overlaps[^1].EndAddress > endAddress)
{
overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress;
overlaps[0].Value.Address = endAddress;
}
else
{
RemoveAt(result.StartIndex);
}
}
if (extendsPre && extendsPost)
{
Add(buffPost);
}
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPre = null;
if (overlaps[0].Address < address)
{
buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address,
overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
extendsPre = true;
}
if (overlaps[^1].EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress,
overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent);
extendsPost = true;
}
RemoveRange(result);
if (extendsPre)
{
Add(buffPre);
}
if (extendsPost)
{
Add(buffPost);
}
Lock.ExitWriteLock();
} }
/// <summary> /// <summary>
@ -549,10 +654,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
public void Clear() public void Clear()
{ {
lock (_lock) Lock.EnterWriteLock();
{ Count = 0;
Count = 0; Lock.ExitWriteLock();
}
} }
} }
} }

7
src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
View file

@ -690,11 +690,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (_pageTable[l0] == null) if (_pageTable[l0] == null)
{ {
_pageTable[l0] = new ulong[PtLvl1Size]; _pageTable[l0] = new ulong[PtLvl1Size];
for (ulong index = 0; index < PtLvl1Size; index++) Array.Fill(_pageTable[l0], PteUnmapped);
{
_pageTable[l0][index] = PteUnmapped;
}
} }
_pageTable[l0][l1] = pte; _pageTable[l0][l1] = pte;

38
src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs
View file

@ -63,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
} }
private readonly RangeList<VirtualRange> _virtualRanges; private readonly RangeList<VirtualRange> _virtualRanges;
private VirtualRange[] _virtualRangeOverlaps; private RangeItem<VirtualRange>[] _virtualRangeOverlaps;
private readonly ConcurrentQueue<VirtualRange> _deferredUnmaps; private readonly ConcurrentQueue<VirtualRange> _deferredUnmaps;
private int _hasDeferredUnmaps; private int _hasDeferredUnmaps;
@ -75,7 +75,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
_memoryManager = memoryManager; _memoryManager = memoryManager;
_virtualRanges = []; _virtualRanges = [];
_virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity]; _virtualRangeOverlaps = new RangeItem<VirtualRange>[BufferCache.OverlapsBufferInitialCapacity];
_deferredUnmaps = new ConcurrentQueue<VirtualRange>(); _deferredUnmaps = new ConcurrentQueue<VirtualRange>();
} }
@ -106,18 +106,17 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>True if the range already existed, false if a new one was created and added</returns> /// <returns>True if the range already existed, false if a new one was created and added</returns>
public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range) public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range)
{ {
VirtualRange[] overlaps = _virtualRangeOverlaps; OverlapResult result;
int overlapsCount;
if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0) if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0)
{ {
while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange)) while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange))
{ {
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps); result = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size);
for (int index = 0; index < overlapsCount; index++) if (result.StartIndex >= 0)
{ {
_virtualRanges.Remove(overlaps[index]); _virtualRanges.RemoveRange(result);
} }
} }
} }
@ -126,27 +125,23 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong originalVa = gpuVa; ulong originalVa = gpuVa;
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps); _virtualRanges.Lock.EnterWriteLock();
result = _virtualRanges.FindOverlapsNonOverlappingAsSpan(gpuVa, size, out ReadOnlySpan<RangeItem<VirtualRange>> overlaps);
if (overlapsCount != 0)
if (overlaps.Length != 0)
{ {
// The virtual range already exists. We just need to check if our range fits inside // The virtual range already exists. We just need to check if our range fits inside
// the existing one, and if not, we must extend the existing one. // the existing one, and if not, we must extend the existing one.
ulong endAddress = gpuVa + size; ulong endAddress = gpuVa + size;
VirtualRange overlap0 = overlaps[0]; VirtualRange overlap0 = overlaps[0].Value;
if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress) if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress)
{ {
for (int index = 0; index < overlapsCount; index++) gpuVa = Math.Min(gpuVa, overlaps[0].Address);
{ endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
VirtualRange virtualRange = overlaps[index];
_virtualRanges.RemoveRange(result);
gpuVa = Math.Min(gpuVa, virtualRange.Address);
endAddress = Math.Max(endAddress, virtualRange.EndAddress);
_virtualRanges.Remove(virtualRange);
}
ulong newSize = endAddress - gpuVa; ulong newSize = endAddress - gpuVa;
MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize); MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize);
@ -170,6 +165,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_virtualRanges.Add(virtualRange); _virtualRanges.Add(virtualRange);
} }
_virtualRanges.Lock.ExitWriteLock();
ShrinkOverlapsBufferIfNeeded(); ShrinkOverlapsBufferIfNeeded();

15
src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs
View file

@ -19,16 +19,18 @@ namespace Ryujinx.Memory.Range
/// <param name="address">Start address of the search region</param> /// <param name="address">Start address of the search region</param>
/// <param name="size">Size of the search region</param> /// <param name="size">Size of the search region</param>
/// <param name="factory">Factory for creating new ranges</param> /// <param name="factory">Factory for creating new ranges</param>
public void GetOrAddRegions(List<T> list, ulong address, ulong size, Func<ulong, ulong, T> factory) public void GetOrAddRegions(out List<T> list, ulong address, ulong size, Func<ulong, ulong, T> factory)
{ {
// (regarding the specific case this generalized function is used for) // (regarding the specific case this generalized function is used for)
// A new region may be split into multiple parts if multiple virtual regions have mapped to it. // A new region may be split into multiple parts if multiple virtual regions have mapped to it.
// For instance, while a virtual mapping could cover 0-2 in physical space, the space 0-1 may have already been reserved... // For instance, while a virtual mapping could cover 0-2 in physical space, the space 0-1 may have already been reserved...
// So we need to return both the split 0-1 and 1-2 ranges. // So we need to return both the split 0-1 and 1-2 ranges.
T[] results = new T[1]; RangeItem<T>[] results = new RangeItem<T>[1];
int count = FindOverlapsNonOverlapping(address, size, ref results); int count = FindOverlapsNonOverlapping(address, size, ref results).Count;
list = new List<T>(count);
Lock.EnterWriteLock();
if (count == 0) if (count == 0)
{ {
// The region is fully unmapped. Create and add it to the range list. // The region is fully unmapped. Create and add it to the range list.
@ -43,11 +45,12 @@ namespace Ryujinx.Memory.Range
for (int i = 0; i < count; i++) for (int i = 0; i < count; i++)
{ {
T region = results[i]; T region = results[i].Value;
if (count == 1 && region.Address == address && region.Size == size) if (count == 1 && region.Address == address && region.Size == size)
{ {
// Exact match, no splitting required. // Exact match, no splitting required.
list.Add(region); list.Add(region);
Lock.ExitWriteLock();
return; return;
} }
@ -85,6 +88,8 @@ namespace Ryujinx.Memory.Range
Add(fillRegion); Add(fillRegion);
} }
} }
Lock.ExitWriteLock();
} }
/// <summary> /// <summary>

480
src/Ryujinx.Memory/Range/RangeList.cs
View file

@ -2,44 +2,63 @@ using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Threading;
namespace Ryujinx.Memory.Range namespace Ryujinx.Memory.Range
{ {
public readonly struct RangeItem<TValue>(TValue value) where TValue : IRange
{
public readonly ulong Address = value.Address;
public readonly ulong EndAddress = value.Address + value.Size;
public readonly TValue Value = value;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool OverlapsWith(ulong address, ulong endAddress)
{
return Address < endAddress && address < EndAddress;
}
}
/// <summary>
/// Result of an Overlaps Finder function.
/// </summary>
/// <remarks>
/// startIndex is inclusive.
/// endIndex is exclusive.
/// </remarks>
public readonly struct OverlapResult
{
public readonly int StartIndex = -1;
public readonly int EndIndex = -1;
public int Count => EndIndex - StartIndex;
public OverlapResult(int startIndex, int endIndex)
{
this.StartIndex = startIndex;
this.EndIndex = endIndex;
}
}
/// <summary> /// <summary>
/// Sorted list of ranges that supports binary search. /// Sorted list of ranges that supports binary search.
/// </summary> /// </summary>
/// <typeparam name="T">Type of the range.</typeparam> /// <typeparam name="T">Type of the range.</typeparam>
public class RangeList<T> : IEnumerable<T> where T : IRange public class RangeList<T> : IEnumerable<T> where T : IRange
{ {
private readonly struct RangeItem<TValue> where TValue : IRange
{
public readonly ulong Address;
public readonly ulong EndAddress;
public readonly TValue Value;
public RangeItem(TValue value)
{
Value = value;
Address = value.Address;
EndAddress = value.Address + value.Size;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool OverlapsWith(ulong address, ulong endAddress)
{
return Address < endAddress && address < EndAddress;
}
}
private const int BackingInitialSize = 1024; private const int BackingInitialSize = 1024;
private const int ArrayGrowthSize = 32;
private RangeItem<T>[] _items; private RangeItem<T>[] _items;
private readonly int _backingGrowthSize; private readonly int _backingGrowthSize;
public int Count { get; protected set; } public int Count { get; protected set; }
public readonly ReaderWriterLockSlim Lock = new();
private const int QuickAccessLength = 8;
private int _offset;
private int _count;
private RangeItem<T>[] _quickAccess = new RangeItem<T>[QuickAccessLength];
/// <summary> /// <summary>
/// Creates a new range list. /// Creates a new range list.
@ -72,22 +91,21 @@ namespace Ryujinx.Memory.Range
/// </summary> /// </summary>
/// <param name="item">The item to be updated</param> /// <param name="item">The item to be updated</param>
/// <returns>True if the item was located and updated, false otherwise</returns> /// <returns>True if the item was located and updated, false otherwise</returns>
public bool Update(T item) protected bool Update(T item)
{ {
int index = BinarySearch(item.Address); int index = BinarySearch(item.Address);
if (index >= 0) if (index >= 0)
{ {
while (index > 0 && _items[index - 1].Address == item.Address)
{
index--;
}
while (index < Count) while (index < Count)
{ {
if (_items[index].Value.Equals(item)) if (_items[index].Value.Equals(item))
{ {
_items[index] = new RangeItem<T>(item); _items[index] = new RangeItem<T>(item);
_quickAccess = new RangeItem<T>[QuickAccessLength];
_count = 0;
_offset = 0;
return true; return true;
} }
@ -129,12 +147,34 @@ namespace Ryujinx.Memory.Range
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveAt(int index) protected void RemoveAt(int index)
{ {
if (index < --Count) if (index < --Count)
{ {
Array.Copy(_items, index + 1, _items, index, Count - index); Array.Copy(_items, index + 1, _items, index, Count - index);
} }
_quickAccess = new RangeItem<T>[QuickAccessLength];
_count = 0;
_offset = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void RemoveRange(OverlapResult overlapResult)
{
if (overlapResult.EndIndex < Count)
{
Array.Copy(_items, overlapResult.EndIndex, _items, overlapResult.StartIndex, Count - overlapResult.EndIndex);
Count -= overlapResult.Count;
}
else if (overlapResult.EndIndex == Count)
{
Count = overlapResult.StartIndex;
}
_quickAccess = new RangeItem<T>[QuickAccessLength];
_count = 0;
_offset = 0;
} }
/// <summary> /// <summary>
@ -148,11 +188,6 @@ namespace Ryujinx.Memory.Range
if (index >= 0) if (index >= 0)
{ {
while (index > 0 && _items[index - 1].Address == item.Address)
{
index--;
}
while (index < Count) while (index < Count)
{ {
if (_items[index].Value.Equals(item)) if (_items[index].Value.Equals(item))
@ -175,67 +210,39 @@ namespace Ryujinx.Memory.Range
} }
/// <summary> /// <summary>
/// Updates an item's end address. /// Gets an item on the list overlapping the specified memory range.
/// </summary>
/// <param name="item">The item to be updated</param>
public void UpdateEndAddress(T item)
{
int index = BinarySearch(item.Address);
if (index >= 0)
{
while (index > 0 && _items[index - 1].Address == item.Address)
{
index--;
}
while (index < Count)
{
if (_items[index].Value.Equals(item))
{
_items[index] = new RangeItem<T>(item);
return;
}
if (_items[index].Address > item.Address)
{
break;
}
index++;
}
}
}
/// <summary>
/// Gets the first item on the list overlapping in memory with the specified item.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// Despite the name, this has no ordering guarantees of the returned item. /// This has no ordering guarantees of the returned item.
/// It only ensures that the item returned overlaps the specified item.
/// </remarks>
/// <param name="item">Item to check for overlaps</param>
/// <returns>The overlapping item, or the default value for the type if none found</returns>
public T FindFirstOverlap(T item)
{
return FindFirstOverlap(item.Address, item.Size);
}
/// <summary>
/// Gets the first item on the list overlapping the specified memory range.
/// </summary>
/// <remarks>
/// Despite the name, this has no ordering guarantees of the returned item.
/// It only ensures that the item returned overlaps the specified memory range. /// It only ensures that the item returned overlaps the specified memory range.
/// </remarks> /// </remarks>
/// <param name="address">Start address of the range</param> /// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param> /// <param name="size">Size in bytes of the range</param>
/// <returns>The overlapping item, or the default value for the type if none found</returns> /// <returns>The overlapping item, or the default value for the type if none found</returns>
public T FindFirstOverlap(ulong address, ulong size) [MethodImpl(MethodImplOptions.AggressiveInlining)]
public T FindOverlapFast(ulong address, ulong size)
{ {
for (int i = 0; i < _quickAccess.Length; i++)
{
ref RangeItem<T> item = ref _quickAccess[(i + _offset) % _quickAccess.Length];
if (item.OverlapsWith(address, address + size))
{
return item.Value;
}
}
int index = BinarySearch(address, address + size); int index = BinarySearch(address, address + size);
if (_count < _quickAccess.Length)
{
_quickAccess[_count++] = _items[index];
}
else
{
_quickAccess[_offset++ % _quickAccess.Length] = _items[index];
}
if (index < 0) if (index < 0)
{ {
return default; return default;
@ -243,68 +250,66 @@ namespace Ryujinx.Memory.Range
return _items[index].Value; return _items[index].Value;
} }
/// <summary>
/// Gets all items overlapping with the specified item in memory.
/// </summary>
/// <param name="item">Item to check for overlaps</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of overlapping items found</returns>
public int FindOverlaps(T item, ref T[] output)
{
return FindOverlaps(item.Address, item.Size, ref output);
}
/// <summary> /// <summary>
/// Gets all items on the list overlapping the specified memory range. /// Gets all items on the list overlapping the specified memory range.
/// </summary> /// </summary>
/// <param name="address">Start address of the range</param> /// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param> /// <param name="size">Size in bytes of the range</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param> /// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of overlapping items found</returns> /// <returns>Range information of overlapping items found</returns>
public int FindOverlaps(ulong address, ulong size, ref T[] output) public OverlapResult FindOverlaps(ulong address, ulong size, ref RangeItem<T>[] output)
{ {
int outputIndex = 0; int outputIndex = 0;
ulong endAddress = address + size; ulong endAddress = address + size;
int startIndex = BinarySearch(address);
if (startIndex < 0)
startIndex = ~startIndex;
int endIndex = -1;
for (int i = 0; i < Count; i++) for (int i = startIndex; i < Count; i++)
{ {
ref RangeItem<T> item = ref _items[i]; ref RangeItem<T> item = ref _items[i];
if (item.Address >= endAddress) if (item.Address >= endAddress)
{ {
endIndex = i;
break; break;
} }
if (item.OverlapsWith(address, endAddress)) if (item.OverlapsWith(address, endAddress))
{ {
if (outputIndex == output.Length) outputIndex++;
{
Array.Resize(ref output, outputIndex + ArrayGrowthSize);
}
output[outputIndex++] = item.Value;
} }
} }
return outputIndex; if (endIndex == -1 && outputIndex > 0)
} {
endIndex = Count;
}
/// <summary> if (outputIndex > 0 && outputIndex == endIndex - startIndex)
/// Gets all items overlapping with the specified item in memory. {
/// </summary> Array.Resize(ref output, outputIndex);
/// <remarks> Array.Copy(_items, endIndex - outputIndex, output, 0, outputIndex);
/// This method only returns correct results if none of the items on the list overlaps with
/// each other. If that is not the case, this method should not be used. return new OverlapResult(startIndex, endIndex);
/// This method is faster than the regular method to find all overlaps. }
/// </remarks> else if (outputIndex > 0)
/// <param name="item">Item to check for overlaps</param> {
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param> Array.Resize(ref output, outputIndex);
/// <returns>The number of overlapping items found</returns> int arrIndex = 0;
public int FindOverlapsNonOverlapping(T item, ref T[] output) for (int i = startIndex; i < endIndex; i++)
{ {
return FindOverlapsNonOverlapping(item.Address, item.Size, ref output); output[arrIndex++] = _items[i];
}
return new OverlapResult(endIndex - outputIndex, endIndex);
}
return new OverlapResult();
} }
/// <summary> /// <summary>
@ -318,77 +323,68 @@ namespace Ryujinx.Memory.Range
/// <param name="address">Start address of the range</param> /// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param> /// <param name="size">Size in bytes of the range</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param> /// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of overlapping items found</returns> /// <returns>Range information of overlapping items found</returns>
public int FindOverlapsNonOverlapping(ulong address, ulong size, ref T[] output) [MethodImpl(MethodImplOptions.AggressiveInlining)]
public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size, ref RangeItem<T>[] output)
{ {
// This is a bit faster than FindOverlaps, but only works // This is a bit faster than FindOverlaps, but only works
// when none of the items on the list overlaps with each other. // when none of the items on the list overlaps with each other.
int outputIndex = 0;
ulong endAddress = address + size; ulong endAddress = address + size;
int index = BinarySearch(address, endAddress); (int index, int endIndex) = BinarySearchEdges(address, endAddress);
if (index >= 0) if (index >= 0)
{ {
while (index > 0 && _items[index - 1].OverlapsWith(address, endAddress)) Array.Resize(ref output, endIndex - index);
{ Array.Copy(_items, index, output, 0, endIndex - index);
index--;
}
do
{
if (outputIndex == output.Length)
{
Array.Resize(ref output, outputIndex + ArrayGrowthSize);
}
output[outputIndex++] = _items[index++].Value;
}
while (index < Count && _items[index].OverlapsWith(address, endAddress));
} }
return outputIndex; return new OverlapResult(index, endIndex);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public OverlapResult FindOverlapsNonOverlappingAsSpan(ulong address, ulong size, out ReadOnlySpan<RangeItem<T>> span)
{
// This is a bit faster than FindOverlaps, but only works
// when none of the items on the list overlaps with each other.
ulong endAddress = address + size;
(int index, int endIndex) = BinarySearchEdges(address, endAddress);
if (index >= 0)
{
span = new ReadOnlySpan<RangeItem<T>>(_items, index, endIndex - index);
return new OverlapResult(index, endIndex);
}
span = ReadOnlySpan<RangeItem<T>>.Empty;
return new OverlapResult(index, endIndex);
} }
/// <summary> /// <summary>
/// Gets all items on the list with the specified memory address. /// Gets the range of all items on the list overlapping the specified memory range.
/// </summary> /// </summary>
/// <param name="address">Address to find</param> /// <remarks>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param> /// This method only returns correct results if none of the items on the list overlaps with
/// <returns>The number of matches found</returns> /// each other. If that is not the case, this method should not be used.
public int FindOverlaps(ulong address, ref T[] output) /// This method is faster than the regular method to find all overlaps.
/// </remarks>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param>
/// <returns>Range information of overlapping items found</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size)
{ {
int index = BinarySearch(address); // This is a bit faster than FindOverlaps, but only works
// when none of the items on the list overlaps with each other.
int outputIndex = 0; ulong endAddress = address + size;
if (index >= 0) (int index, int endIndex) = BinarySearchEdges(address, endAddress);
{
while (index > 0 && _items[index - 1].Address == address)
{
index--;
}
while (index < Count) return new OverlapResult(index, endIndex);
{
ref RangeItem<T> overlap = ref _items[index++];
if (overlap.Address != address)
{
break;
}
if (outputIndex == output.Length)
{
Array.Resize(ref output, outputIndex + ArrayGrowthSize);
}
output[outputIndex++] = overlap.Value;
}
}
return outputIndex;
} }
/// <summary> /// <summary>
@ -396,6 +392,7 @@ namespace Ryujinx.Memory.Range
/// </summary> /// </summary>
/// <param name="address">Address to find</param> /// <param name="address">Address to find</param>
/// <returns>List index of the item, or complement index of nearest item with lower value on the list</returns> /// <returns>List index of the item, or complement index of nearest item with lower value on the list</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int BinarySearch(ulong address) private int BinarySearch(ulong address)
{ {
int left = 0; int left = 0;
@ -426,13 +423,14 @@ namespace Ryujinx.Memory.Range
return ~left; return ~left;
} }
/// <summary> /// <summary>
/// Performs binary search for items overlapping a given memory range. /// Performs binary search for items overlapping a given memory range.
/// </summary> /// </summary>
/// <param name="address">Start address of the range</param> /// <param name="address">Start address of the range</param>
/// <param name="endAddress">End address of the range</param> /// <param name="endAddress">End address of the range</param>
/// <returns>List index of the item, or complement index of nearest item with lower value on the list</returns> /// <returns>List index of the item, or complement index of nearest item with lower value on the list</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int BinarySearch(ulong address, ulong endAddress) private int BinarySearch(ulong address, ulong endAddress)
{ {
int left = 0; int left = 0;
@ -464,6 +462,136 @@ namespace Ryujinx.Memory.Range
return ~left; return ~left;
} }
/// <summary>
/// Performs binary search for items overlapping a given memory range.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="endAddress">End address of the range</param>
/// <returns>Range information (inclusive, exclusive) of items that overlaps, or complement index of nearest item with lower value on the list</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private (int, int) BinarySearchEdges(ulong address, ulong endAddress)
{
if (Count == 0)
return (~0, ~0);
if (Count == 1)
{
ref RangeItem<T> item = ref _items[0];
if (item.OverlapsWith(address, endAddress))
{
return (0, 1);
}
if (address < item.Address)
{
return (~0, ~0);
}
else
{
return (~1, ~1);
}
}
int left = 0;
int right = Count - 1;
int leftEdge = -1;
int rightEdgeMatch = -1;
int rightEdgeNoMatch = -1;
while (left <= right)
{
int range = right - left;
int middle = left + (range >> 1);
ref RangeItem<T> item = ref _items[middle];
bool match = item.OverlapsWith(address, endAddress);
if (range == 0)
{
if (match)
{
leftEdge = middle;
break;
}
else if (address < item.Address)
{
return (~right, ~right);
}
else
{
return (~(right + 1), ~(right + 1));
}
}
if (match)
{
right = middle;
if (rightEdgeMatch == -1)
rightEdgeMatch = middle;
}
else if (address < item.Address)
{
right = middle - 1;
rightEdgeNoMatch = middle;
}
else
{
left = middle + 1;
}
}
if (left > right)
{
return (~left, ~left);
}
if (rightEdgeMatch == -1)
{
return (leftEdge, leftEdge + 1);
}
left = rightEdgeMatch;
right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1;
while (left <= right)
{
int range = right - left;
int middle = right - (range >> 1);
ref RangeItem<T> item = ref _items[middle];
bool match = item.OverlapsWith(address, endAddress);
if (range == 0)
{
if (match)
return (leftEdge, middle + 1);
else
return (leftEdge, middle);
}
if (match)
{
left = middle;
}
else if (address < item.Address)
{
right = middle - 1;
}
else
{
left = middle + 1;
}
}
return (leftEdge, right + 1);
}
public IEnumerator<T> GetEnumerator() public IEnumerator<T> GetEnumerator()
{ {
for (int i = 0; i < Count; i++) for (int i = 0; i < Count; i++)

55
src/Ryujinx.Memory/Tracking/MemoryTracking.cs
View file

@ -1,5 +1,6 @@
using Ryujinx.Common.Pools; using Ryujinx.Common.Pools;
using Ryujinx.Memory.Range; using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic; using System.Collections.Generic;
namespace Ryujinx.Memory.Tracking namespace Ryujinx.Memory.Tracking
@ -76,17 +77,15 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock) lock (TrackingLock)
{ {
ref VirtualRegion[] overlaps = ref ThreadStaticArray<VirtualRegion>.Get();
for (int type = 0; type < 2; type++) for (int type = 0; type < 2; type++)
{ {
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions; NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock();
int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan<RangeItem<VirtualRegion>> overlaps);
for (int i = 0; i < count; i++) for (int i = 0; i < overlaps.Length; i++)
{ {
VirtualRegion region = overlaps[i]; VirtualRegion region = overlaps[i].Value;
// If the region has been fully remapped, signal that it has been mapped again. // If the region has been fully remapped, signal that it has been mapped again.
bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size);
@ -97,6 +96,7 @@ namespace Ryujinx.Memory.Tracking
region.UpdateProtection(); region.UpdateProtection();
} }
regions.Lock.ExitReadLock();
} }
} }
} }
@ -114,20 +114,19 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock) lock (TrackingLock)
{ {
ref VirtualRegion[] overlaps = ref ThreadStaticArray<VirtualRegion>.Get();
for (int type = 0; type < 2; type++) for (int type = 0; type < 2; type++)
{ {
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions; NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock();
regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan<RangeItem<VirtualRegion>> overlaps);
int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); for (int i = 0; i < overlaps.Length; i++)
for (int i = 0; i < count; i++)
{ {
VirtualRegion region = overlaps[i]; VirtualRegion region = overlaps[i].Value;
region.SignalMappingChanged(false); region.SignalMappingChanged(false);
} }
regions.Lock.ExitReadLock();
} }
} }
} }
@ -165,10 +164,11 @@ namespace Ryujinx.Memory.Tracking
/// <returns>A list of virtual regions within the given range</returns> /// <returns>A list of virtual regions within the given range</returns>
internal List<VirtualRegion> GetVirtualRegionsForHandle(ulong va, ulong size, bool guest) internal List<VirtualRegion> GetVirtualRegionsForHandle(ulong va, ulong size, bool guest)
{ {
List<VirtualRegion> result = [];
NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions; NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions;
regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); regions.Lock.EnterUpgradeableReadLock();
regions.GetOrAddRegions(out List<VirtualRegion> result, va, size, (va, size) => new VirtualRegion(this, va, size, guest));
regions.Lock.ExitUpgradeableReadLock();
return result; return result;
} }
@ -296,25 +296,26 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock) lock (TrackingLock)
{ {
ref VirtualRegion[] overlaps = ref ThreadStaticArray<VirtualRegion>.Get();
NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions; NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions;
ref RangeItem<VirtualRegion>[] overlaps = ref ThreadStaticArray<RangeItem<VirtualRegion>>.Get();
// We use the non-span method here because keeping the lock will cause a deadlock.
regions.Lock.EnterReadLock();
OverlapResult result = regions.FindOverlapsNonOverlapping(address, size, ref overlaps);
regions.Lock.ExitReadLock();
int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); if (overlaps.Length == 0 && !precise)
if (count == 0 && !precise)
{ {
if (_memoryManager.IsRangeMapped(address, size)) if (_memoryManager.IsRangeMapped(address, size))
{ {
// TODO: There is currently the possibility that a page can be protected after its virtual region is removed. // TODO: There is currently the possibility that a page can be protected after its virtual region is removed.
// This code handles that case when it happens, but it would be better to find out how this happens. // This code handles that case when it happens, but it would be better to find out how this happens.
_memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest); _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest);
return true; // This memory _should_ be mapped, so we need to try again. return true; // This memory _should_ be mapped, so we need to try again.
} }
else
{ shouldThrow = true;
shouldThrow = true;
}
} }
else else
{ {
@ -324,9 +325,9 @@ namespace Ryujinx.Memory.Tracking
size += (ulong)_pageSize; size += (ulong)_pageSize;
} }
for (int i = 0; i < count; i++) for (int i = 0; i < result.Count; i++)
{ {
VirtualRegion region = overlaps[i]; VirtualRegion region = overlaps[i].Value;
if (precise) if (precise)
{ {