crafterpika/ryujinx
1
0
Fork 0
mirror of https://git.ryujinx.app/ryubing/ryujinx.git synced 2025-06-28 00:16:23 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'memory-refactor' into 'master'

Browse source 
Memory RangeList Refactor

See merge request [ryubing/ryujinx!46](https://git.ryujinx.app/ryubing/ryujinx/-/merge_requests/46)
This commit is contained in:
LotP 2025-06-21 11:40:19 -05:00
commit ccf7bc9e0d
10 changed files with 723 additions and 506 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View file

@ -110,7 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong size,
BufferStage stage,
bool sparseCompatible,
IEnumerable<Buffer> baseBuffers = null)
ReadOnlySpan<RangeItem<Buffer>> baseBuffers)
{
_context = context;
_physicalMemory = physicalMemory;
@ -126,21 +126,22 @@ namespace Ryujinx.Graphics.Gpu.Memory
_useGranular = size > GranularBufferThreshold;
IEnumerable<IRegionHandle> baseHandles = null;
List<IRegionHandle> baseHandles = null;
if (baseBuffers != null)
if (!baseBuffers.IsEmpty)
{
baseHandles = baseBuffers.SelectMany(buffer =>
baseHandles = new List<IRegionHandle>();
foreach (RangeItem<Buffer> buffer in baseBuffers)
{
if (buffer._useGranular)
if (buffer.Value._useGranular)
{
return buffer._memoryTrackingGranular.GetHandles();
baseHandles.AddRange((buffer.Value._memoryTrackingGranular.GetHandles()));
}
else
{
return Enumerable.Repeat(buffer._memoryTracking, 1);
baseHandles.Add(buffer.Value._memoryTracking);
}
});
}
}
if (_useGranular)

11
src/Ryujinx.Graphics.Gpu/Memory/BufferBackingState.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
@ -56,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="parent">Parent buffer</param>
/// <param name="stage">Initial buffer stage</param>
/// <param name="baseBuffers">Buffers to inherit state from</param>
public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, IEnumerable<Buffer> baseBuffers = null)
public BufferBackingState(GpuContext context, Buffer parent, BufferStage stage, ReadOnlySpan<RangeItem<Buffer>> baseBuffers)
{
_size = (int)parent.Size;
_systemMemoryType = context.Capabilities.MemoryType;
@ -72,7 +73,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferStage storageFlags = stage & BufferStage.StorageMask;
if (parent.Size > DeviceLocalSizeThreshold && baseBuffers == null)
if (parent.Size > DeviceLocalSizeThreshold && baseBuffers.IsEmpty)
{
_desiredType = BufferBackingType.DeviceMemory;
}
@ -100,11 +101,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
// TODO: Might be nice to force atomic access to be device local for any stage.
}
if (baseBuffers != null)
if (!baseBuffers.IsEmpty)
{
foreach (Buffer buffer in baseBuffers)
foreach (RangeItem<Buffer> buffer in baseBuffers)
{
CombineState(buffer.BackingState);
CombineState(buffer.Value.BackingState);
}
}
}

134
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
View file

@ -2,7 +2,6 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
namespace Ryujinx.Graphics.Gpu.Memory
@ -42,7 +41,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
private readonly RangeList<Buffer> _buffers;
private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers;
private Buffer[] _bufferOverlaps;
private RangeItem<Buffer>[] _bufferOverlaps;
private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache;
private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache;
@ -64,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_buffers = [];
_multiRangeBuffers = [];
_bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity];
_bufferOverlaps = new RangeItem<Buffer>[OverlapsBufferInitialCapacity];
_dirtyCache = new Dictionary<ulong, BufferCacheEntry>();
@ -79,7 +78,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="e">Event arguments</param>
public void MemoryUnmappedHandler(object sender, UnmapEventArgs e)
{
Buffer[] overlaps = new Buffer[10];
RangeItem<Buffer>[] overlaps = new RangeItem<Buffer>[10];
int overlapCount;
MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
@ -90,12 +89,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
lock (_buffers)
{
overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps);
overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps).Count;
}
for (int i = 0; i < overlapCount; i++)
{
overlaps[i].Unmapped(subRange.Address, subRange.Size);
overlaps[i].Value.Unmapped(subRange.Address, subRange.Size);
}
}
}
@ -336,7 +335,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
ulong alignedSize = alignedEndAddress - alignedAddress;
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
Buffer buffer = _buffers.FindOverlapFast(alignedAddress, alignedSize);
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
@ -403,7 +402,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (subRange.Address != MemoryManager.PteUnmapped)
{
Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
Buffer buffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size);
virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size);
physicalBuffers.Add(buffer);
@ -495,10 +494,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="stage">The type of usage that created the buffer</param>
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage)
{
Buffer[] overlaps = _bufferOverlaps;
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
_buffers.Lock.EnterWriteLock();
OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<Buffer>> overlaps);
if (overlapsCount != 0)
if (result.Count != 0)
{
// The buffer already exists. We can just return the existing buffer
// if the buffer we need is fully contained inside the overlapping buffer.
@ -507,7 +506,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
// old buffer(s) to the new buffer.
ulong endAddress = address + size;
Buffer overlap0 = overlaps[0];
RangeItem<Buffer> overlap0 = overlaps[0];
if (overlap0.Address > address || overlap0.EndAddress < endAddress)
{
@ -522,51 +521,51 @@ namespace Ryujinx.Graphics.Gpu.Memory
// sequential memory.
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages.
if (overlapsCount == 1 &&
if (result.Count == 1 &&
address >= overlap0.Address &&
endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2)
{
// Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts.
ulong existingSize = overlap0.Size;
ulong existingSize = overlap0.Value.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize);
endAddress = address + size;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out overlaps);
}
address = Math.Min(address, overlaps[0].Address);
endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
for (int index = 0; index < overlapsCount; index++)
for (int index = 0; index < result.Count; index++)
{
Buffer buffer = overlaps[index];
RangeItem<Buffer> buffer = overlaps[index];
anySparseCompatible |= buffer.SparseCompatible;
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
lock (_buffers)
{
_buffers.Remove(buffer);
}
anySparseCompatible |= buffer.Value.SparseCompatible;
}
ulong newSize = endAddress - address;
CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps, overlapsCount);
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, anySparseCompatible, overlaps);
_buffers.RemoveRange(result);
_buffers.Add(newBuffer);
}
}
else
{
// No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false);
Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible: false, ReadOnlySpan<RangeItem<Buffer>>.Empty);
lock (_buffers)
{
_buffers.Add(buffer);
}
_buffers.Add(buffer);
}
_buffers.Lock.ExitWriteLock();
ShrinkOverlapsBufferIfNeeded();
}
@ -582,22 +581,22 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="alignment">Alignment of the start address of the buffer</param>
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, ulong alignment)
{
Buffer[] overlaps = _bufferOverlaps;
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
_buffers.Lock.EnterWriteLock();
OverlapResult result = _buffers.FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<Buffer>> overlaps);
bool sparseAligned = alignment >= SparseBufferAlignmentSize;
if (overlapsCount != 0)
if (result.Count != 0)
{
// If the buffer already exists, make sure if covers the entire range,
// and make sure it is properly aligned, otherwise sparse mapping may fail.
ulong endAddress = address + size;
Buffer overlap0 = overlaps[0];
RangeItem<Buffer> overlap0 = overlaps[0];
if (overlap0.Address > address ||
overlap0.EndAddress < endAddress ||
(overlap0.Address & (alignment - 1)) != 0 ||
(!overlap0.SparseCompatible && sparseAligned))
(!overlap0.Value.SparseCompatible && sparseAligned))
{
// We need to make sure the new buffer is properly aligned.
// However, after the range is aligned, it is possible that it
@ -605,47 +604,36 @@ namespace Ryujinx.Graphics.Gpu.Memory
// and ensure we cover all overlaps.
int oldOverlapsCount;
endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
do
{
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = overlaps[index];
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
}
address = Math.Min(address, overlaps[0].Address);
address &= ~(alignment - 1);
oldOverlapsCount = overlapsCount;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps);
}
while (oldOverlapsCount != overlapsCount);
lock (_buffers)
{
for (int index = 0; index < overlapsCount; index++)
{
_buffers.Remove(overlaps[index]);
}
oldOverlapsCount = result.Count;
result = _buffers.FindOverlapsNonOverlappingAsSpan(address, endAddress - address, out overlaps);
}
while (oldOverlapsCount != result.Count);
ulong newSize = endAddress - address;
CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps, overlapsCount);
Buffer newBuffer = CreateBufferAligned(address, newSize, stage, sparseAligned, overlaps);
_buffers.RemoveRange(result);
_buffers.Add(newBuffer);
}
}
else
{
// No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned);
Buffer buffer = new(_context, _physicalMemory, address, size, stage, sparseAligned, ReadOnlySpan<RangeItem<Buffer>>.Empty);
lock (_buffers)
{
_buffers.Add(buffer);
}
_buffers.Add(buffer);
}
_buffers.Lock.ExitWriteLock();
ShrinkOverlapsBufferIfNeeded();
}
@ -660,19 +648,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="stage">The type of usage that created the buffer</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="overlaps">Buffers overlapping the range</param>
/// <param name="overlapsCount">Total of overlaps</param>
private void CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, Buffer[] overlaps, int overlapsCount)
private Buffer CreateBufferAligned(ulong address, ulong size, BufferStage stage, bool sparseCompatible, ReadOnlySpan<RangeItem<Buffer>> overlaps)
{
Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps.Take(overlapsCount));
Buffer newBuffer = new(_context, _physicalMemory, address, size, stage, sparseCompatible, overlaps);
lock (_buffers)
for (int index = 0; index < overlaps.Length; index++)
{
_buffers.Add(newBuffer);
}
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = overlaps[index];
Buffer buffer = overlaps[index].Value;
int dstOffset = (int)(buffer.Address - newBuffer.Address);
@ -688,6 +670,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
NotifyBuffersModified?.Invoke();
RecreateMultiRangeBuffers(address, size);
return newBuffer;
}
/// <summary>
@ -909,7 +893,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
MemoryRange subRange = range.GetSubRange(i);
Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
Buffer subBuffer = _buffers.FindOverlapFast(subRange.Address, subRange.Size);
subBuffer.SynchronizeMemory(subRange.Address, subRange.Size);
@ -957,7 +941,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (size != 0)
{
buffer = _buffers.FindFirstOverlap(address, size);
buffer = _buffers.FindOverlapFast(address, size);
buffer.CopyFromDependantVirtualBuffers();
buffer.SynchronizeMemory(address, size);
@ -969,7 +953,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
}
else
{
buffer = _buffers.FindFirstOverlap(address, 1);
buffer = _buffers.FindOverlapFast(address, 1);
}
return buffer;
@ -1007,7 +991,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
if (size != 0)
{
Buffer buffer = _buffers.FindFirstOverlap(address, size);
Buffer buffer = _buffers.FindOverlapFast(address, size);
if (copyBackVirtual)
{

4
src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
View file

@ -761,7 +761,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (!bounds.IsUnmapped)
{
bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write;
BufferRange range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Range, bufferStage | BufferStageUtils.FromUsage(bounds.Flags), isWrite)
: bufferCache.GetBufferRange(bounds.Range, bufferStage);
@ -798,7 +798,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (!bounds.IsUnmapped)
{
bool isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
bool isWrite = (bounds.Flags & BufferUsageFlags.Write) == BufferUsageFlags.Write;
BufferRange range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Range, BufferStageUtils.ComputeStorage(bounds.Flags), isWrite)
: bufferCache.GetBufferRange(bounds.Range, BufferStage.Compute);

468
src/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
View file

@ -2,7 +2,6 @@ using Ryujinx.Common.Pools;
using Ryujinx.Memory.Range;
using System;
using System.Linq;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory
{
@ -14,12 +13,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <summary>
/// Start address of the range in guest memory.
/// </summary>
public ulong Address { get; }
public ulong Address { get; internal set; }
/// <summary>
/// Size of the range in bytes.
/// </summary>
public ulong Size { get; }
public ulong Size { get; internal set; }
/// <summary>
/// End address of the range in guest memory.
@ -77,8 +76,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
private BufferMigration _source;
private BufferModifiedRangeList _migrationTarget;
private readonly Lock _lock = new();
/// <summary>
/// Whether the modified range list has any entries or not.
/// </summary>
@ -86,10 +83,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
get
{
lock (_lock)
{
return Count > 0;
}
Lock.EnterReadLock();
bool result = Count > 0;
Lock.ExitReadLock();
return result;
}
}
@ -114,33 +111,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="action">Action to perform for each remaining sub-range of the input range</param>
public void ExcludeModifiedRegions(ulong address, ulong size, Action<ulong, ulong> action)
{
lock (_lock)
// Slices a given region using the modified regions in the list. Calls the action for the new slices.
bool lockOwner = Lock.IsReadLockHeld;
if (!lockOwner)
{
// Slices a given region using the modified regions in the list. Calls the action for the new slices.
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get();
Lock.EnterReadLock();
}
int count = FindOverlapsNonOverlapping(address, size, ref overlaps);
FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
for (int i = 0; i < count; i++)
for (int i = 0; i < overlaps.Length; i++)
{
BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.Address > address)
{
BufferModifiedRange overlap = overlaps[i];
if (overlap.Address > address)
{
// The start of the remaining region is uncovered by this overlap. Call the action for it.
action(address, overlap.Address - address);
}
// Remaining region is after this overlap.
size -= overlap.EndAddress - address;
address = overlap.EndAddress;
// The start of the remaining region is uncovered by this overlap. Call the action for it.
action(address, overlap.Address - address);
}
if ((long)size > 0)
{
// If there is any region left after removing the overlaps, signal it.
action(address, size);
}
// Remaining region is after this overlap.
size -= overlap.EndAddress - address;
address = overlap.EndAddress;
}
if (!lockOwner)
{
Lock.ExitReadLock();
}
if ((long)size > 0)
{
// If there is any region left after removing the overlaps, signal it.
action(address, size);
}
}
@ -152,51 +155,102 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size of the modified region in bytes</param>
public void SignalModified(ulong address, ulong size)
{
// Must lock, as this can affect flushes from the background thread.
lock (_lock)
// We may overlap with some existing modified regions. They must be cut into by the new entry.
Lock.EnterWriteLock();
OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size,
out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
ulong endAddress = address + size;
ulong syncNumber = _context.SyncNumber;
if (overlaps.Length == 0)
{
// We may overlap with some existing modified regions. They must be cut into by the new entry.
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get();
Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
return;
}
int count = FindOverlapsNonOverlapping(address, size, ref overlaps);
BufferModifiedRange buffPost = null;
bool extendsPost = false;
bool extendsPre = false;
ulong endAddress = address + size;
ulong syncNumber = _context.SyncNumber;
for (int i = 0; i < count; i++)
if (overlaps.Length == 1)
{
if (overlaps[0].Address == address && overlaps[0].EndAddress == endAddress)
{
// The overlaps must be removed or split.
overlaps[0].Value.SyncNumber = syncNumber;
overlaps[0].Value.Parent = this;
Lock.ExitWriteLock();
return;
}
BufferModifiedRange overlap = overlaps[i];
if (overlaps[0].Address < address)
{
overlaps[0].Value.Size = address - overlaps[0].Address;
if (overlap.Address == address && overlap.Size == size)
extendsPre = true;
if (overlaps[0].EndAddress > endAddress)
{
// Region already exists. Just update the existing sync number.
overlap.SyncNumber = syncNumber;
overlap.Parent = this;
return;
buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress,
overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
extendsPost = true;
}
Remove(overlap);
if (overlap.Address < address && overlap.EndAddress > address)
}
else
{
if (overlaps[0].EndAddress > endAddress)
{
// A split item must be created behind this overlap.
Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent));
overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress;
overlaps[0].Value.Address = endAddress;
}
if (overlap.Address < endAddress && overlap.EndAddress > endAddress)
else
{
// A split item must be created after this overlap.
Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent));
RemoveAt(result.StartIndex);
}
}
if (extendsPre && extendsPost)
{
Add(buffPost);
}
Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPre = null;
if (overlaps[0].Address < address)
{
buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address,
overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
extendsPre = true;
}
if (overlaps[^1].EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress,
overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent);
extendsPost = true;
}
RemoveRange(result);
if (extendsPre)
{
Add(buffPre);
}
if (extendsPost)
{
Add(buffPost);
}
Add(new BufferModifiedRange(address, size, syncNumber, this));
Lock.ExitWriteLock();
}
/// <summary>
@ -208,25 +262,20 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="rangeAction">The action to call for each modified range</param>
public void GetRangesAtSync(ulong address, ulong size, ulong syncNumber, Action<ulong, ulong> rangeAction)
{
int count = 0;
Lock.EnterReadLock();
FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get();
// Range list must be consistent for this operation.
lock (_lock)
for (int i = 0; i < overlaps.Length; i++)
{
count = FindOverlapsNonOverlapping(address, size, ref overlaps);
}
for (int i = 0; i < count; i++)
{
BufferModifiedRange overlap = overlaps[i];
BufferModifiedRange overlap = overlaps[i].Value;
if (overlap.SyncNumber == syncNumber)
{
rangeAction(overlap.Address, overlap.Size);
}
}
Lock.ExitReadLock();
}
/// <summary>
@ -237,19 +286,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="rangeAction">The action to call for each modified range</param>
public void GetRanges(ulong address, ulong size, Action<ulong, ulong> rangeAction)
{
int count = 0;
RangeItem<BufferModifiedRange>[] overlaps = new RangeItem<BufferModifiedRange>[1];
// We use the non-span method here because keeping the lock will cause a deadlock.
Lock.EnterReadLock();
OverlapResult result = FindOverlapsNonOverlapping(address, size, ref overlaps);
Lock.ExitReadLock();
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get();
// Range list must be consistent for this operation.
lock (_lock)
for (int i = 0; i < result.Count; i++)
{
count = FindOverlapsNonOverlapping(address, size, ref overlaps);
}
for (int i = 0; i < count; i++)
{
BufferModifiedRange overlap = overlaps[i];
BufferModifiedRange overlap = overlaps[i].Value;
rangeAction(overlap.Address, overlap.Size);
}
}
@ -262,11 +308,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>True if a range exists in the specified region, false otherwise</returns>
public bool HasRange(ulong address, ulong size)
{
// Range list must be consistent for this operation.
lock (_lock)
{
return FindOverlapsNonOverlapping(address, size, ref ThreadStaticArray<BufferModifiedRange>.Get()) > 0;
}
Lock.EnterReadLock();
bool result = FindOverlapsNonOverlapping(address, size).Count > 0;
Lock.ExitReadLock();
return result;
}
/// <summary>
@ -298,38 +343,37 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="address">The start address of the flush range</param>
/// <param name="endAddress">The end address of the flush range</param>
private void RemoveRangesAndFlush(
BufferModifiedRange[] overlaps,
RangeItem<BufferModifiedRange>[] overlaps,
int rangeCount,
long highestDiff,
ulong currentSync,
ulong address,
ulong endAddress)
{
lock (_lock)
if (_migrationTarget == null)
{
if (_migrationTarget == null)
ulong waitSync = currentSync + (ulong)highestDiff;
for (int i = 0; i < rangeCount; i++)
{
ulong waitSync = currentSync + (ulong)highestDiff;
BufferModifiedRange overlap = overlaps[i].Value;
for (int i = 0; i < rangeCount; i++)
long diff = (long)(overlap.SyncNumber - currentSync);
if (diff <= highestDiff)
{
BufferModifiedRange overlap = overlaps[i];
ulong clampAddress = Math.Max(address, overlap.Address);
ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
long diff = (long)(overlap.SyncNumber - currentSync);
Lock.EnterWriteLock();
ClearPart(overlap, clampAddress, clampEnd);
Lock.ExitWriteLock();
if (diff <= highestDiff)
{
ulong clampAddress = Math.Max(address, overlap.Address);
ulong clampEnd = Math.Min(endAddress, overlap.EndAddress);
ClearPart(overlap, clampAddress, clampEnd);
RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction);
}
RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, _flushAction);
}
return;
}
return;
}
// There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine.
@ -353,30 +397,31 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size;
ulong currentSync = _context.SyncNumber;
int rangeCount = 0;
int rangeCount;
ref BufferModifiedRange[] overlaps = ref ThreadStaticArray<BufferModifiedRange>.Get();
ref RangeItem<BufferModifiedRange>[] overlaps = ref ThreadStaticArray<RangeItem<BufferModifiedRange>>.Get();
// Range list must be consistent for this operation
lock (_lock)
Lock.EnterReadLock();
if (_migrationTarget != null)
{
if (_migrationTarget != null)
{
rangeCount = -1;
}
else
{
rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps);
}
rangeCount = -1;
}
else
{
// We use the non-span method here because the array is partially modified by the code, which would invalidate a span.
rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps).Count;
}
Lock.ExitReadLock();
if (rangeCount == -1)
{
_migrationTarget.WaitForAndFlushRanges(address, size);
_migrationTarget!.WaitForAndFlushRanges(address, size);
return;
}
else if (rangeCount == 0)
if (rangeCount == 0)
{
return;
}
@ -388,7 +433,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < rangeCount; i++)
{
BufferModifiedRange overlap = overlaps[i];
BufferModifiedRange overlap = overlaps[i].Value;
long diff = (long)(overlap.SyncNumber - currentSync);
@ -419,42 +464,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="registerRangeAction">The action to call for each modified range</param>
public void InheritRanges(BufferModifiedRangeList ranges, Action<ulong, ulong> registerRangeAction)
{
BufferModifiedRange[] inheritRanges;
ranges.Lock.EnterReadLock();
BufferModifiedRange[] inheritRanges = ranges.ToArray();
ranges.Lock.ExitReadLock();
lock (ranges._lock)
// Copy over the migration from the previous range list
BufferMigration oldMigration = ranges._source;
BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration);
ranges._parent.IncrementReferenceCount();
if (_source == null)
{
inheritRanges = ranges.ToArray();
// Create a new migration.
_source = new BufferMigration([span], this, _context.SyncNumber);
lock (_lock)
{
// Copy over the migration from the previous range list
BufferMigration oldMigration = ranges._source;
BufferMigrationSpan span = new(ranges._parent, ranges._flushAction, oldMigration);
ranges._parent.IncrementReferenceCount();
if (_source == null)
{
// Create a new migration.
_source = new BufferMigration([span], this, _context.SyncNumber);
_context.RegisterBufferMigration(_source);
}
else
{
// Extend the migration
_source.AddSpanToEnd(span);
}
ranges._migrationTarget = this;
foreach (BufferModifiedRange range in inheritRanges)
{
Add(range);
}
}
_context.RegisterBufferMigration(_source);
}
else
{
// Extend the migration
_source.AddSpanToEnd(span);
}
ranges._migrationTarget = this;
Lock.EnterWriteLock();
foreach (BufferModifiedRange range in inheritRanges)
{
Add(range);
}
Lock.ExitWriteLock();
ulong currentSync = _context.SyncNumber;
foreach (BufferModifiedRange range in inheritRanges)
@ -473,18 +515,18 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
public void SelfMigration()
{
lock (_lock)
{
BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(), _parent.GetSnapshotFlushAction(), _source);
BufferMigration migration = new([span], this, _context.SyncNumber);
BufferMigrationSpan span = new(_parent, _parent.GetSnapshotDisposeAction(),
_parent.GetSnapshotFlushAction(), _source);
BufferMigration migration = new([span], this, _context.SyncNumber);
// Migration target is used to redirect flush actions to the latest range list,
// so we don't need to set it here. (this range list is still the latest)
// Migration target is used to redirect flush actions to the latest range list,
// so we don't need to set it here. (this range list is still the latest)
_context.RegisterBufferMigration(migration);
_context.RegisterBufferMigration(migration);
_source = migration;
}
Lock.EnterWriteLock();
_source = migration;
Lock.ExitWriteLock();
}
/// <summary>
@ -493,13 +535,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="migration">The migration to remove</param>
public void RemoveMigration(BufferMigration migration)
{
lock (_lock)
Lock.EnterWriteLock();
if (_source == migration)
{
if (_source == migration)
{
_source = null;
}
_source = null;
}
Lock.ExitWriteLock();
}
private void ClearPart(BufferModifiedRange overlap, ulong address, ulong endAddress)
@ -526,22 +568,85 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size to clear</param>
public void Clear(ulong address, ulong size)
{
lock (_lock)
ulong endAddress = address + size;
Lock.EnterWriteLock();
OverlapResult result = FindOverlapsNonOverlappingAsSpan(address, size, out ReadOnlySpan<RangeItem<BufferModifiedRange>> overlaps);
if (overlaps.Length == 0)
{
// This function can be called from any thread, so it cannot use the arrays for background or foreground.
BufferModifiedRange[] toClear = new BufferModifiedRange[1];
int rangeCount = FindOverlapsNonOverlapping(address, size, ref toClear);
ulong endAddress = address + size;
for (int i = 0; i < rangeCount; i++)
{
BufferModifiedRange overlap = toClear[i];
ClearPart(overlap, address, endAddress);
}
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPost = null;
bool extendsPost = false;
bool extendsPre = false;
if (overlaps.Length == 1)
{
if (overlaps[0].Address < address)
{
overlaps[0].Value.Size = address - overlaps[0].Address;
extendsPre = true;
if (overlaps[0].EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, overlaps[0].EndAddress - endAddress,
overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
extendsPost = true;
}
}
else
{
if (overlaps[^1].EndAddress > endAddress)
{
overlaps[0].Value.Size = overlaps[0].EndAddress - endAddress;
overlaps[0].Value.Address = endAddress;
}
else
{
RemoveAt(result.StartIndex);
}
}
if (extendsPre && extendsPost)
{
Add(buffPost);
}
Lock.ExitWriteLock();
return;
}
BufferModifiedRange buffPre = null;
if (overlaps[0].Address < address)
{
buffPre = new BufferModifiedRange(overlaps[0].Address, address - overlaps[0].Address,
overlaps[0].Value.SyncNumber, overlaps[0].Value.Parent);
extendsPre = true;
}
if (overlaps[^1].EndAddress > endAddress)
{
buffPost = new BufferModifiedRange(endAddress, overlaps[^1].EndAddress - endAddress,
overlaps[^1].Value.SyncNumber, overlaps[^1].Value.Parent);
extendsPost = true;
}
RemoveRange(result);
if (extendsPre)
{
Add(buffPre);
}
if (extendsPost)
{
Add(buffPost);
}
Lock.ExitWriteLock();
}
/// <summary>
@ -549,10 +654,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
public void Clear()
{
lock (_lock)
{
Count = 0;
}
Lock.EnterWriteLock();
Count = 0;
Lock.ExitWriteLock();
}
}
}

7
src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
View file

@ -690,11 +690,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (_pageTable[l0] == null)
{
_pageTable[l0] = new ulong[PtLvl1Size];
for (ulong index = 0; index < PtLvl1Size; index++)
{
_pageTable[l0][index] = PteUnmapped;
}
Array.Fill(_pageTable[l0], PteUnmapped);
}
_pageTable[l0][l1] = pte;

38
src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs
View file

@ -63,7 +63,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
}
private readonly RangeList<VirtualRange> _virtualRanges;
private VirtualRange[] _virtualRangeOverlaps;
private RangeItem<VirtualRange>[] _virtualRangeOverlaps;
private readonly ConcurrentQueue<VirtualRange> _deferredUnmaps;
private int _hasDeferredUnmaps;
@ -75,7 +75,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
_memoryManager = memoryManager;
_virtualRanges = [];
_virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity];
_virtualRangeOverlaps = new RangeItem<VirtualRange>[BufferCache.OverlapsBufferInitialCapacity];
_deferredUnmaps = new ConcurrentQueue<VirtualRange>();
}
@ -106,18 +106,17 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>True if the range already existed, false if a new one was created and added</returns>
public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range)
{
VirtualRange[] overlaps = _virtualRangeOverlaps;
int overlapsCount;
OverlapResult result;
if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0)
{
while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange))
{
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps);
result = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size);
for (int index = 0; index < overlapsCount; index++)
if (result.StartIndex >= 0)
{
_virtualRanges.Remove(overlaps[index]);
_virtualRanges.RemoveRange(result);
}
}
}
@ -126,27 +125,23 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong originalVa = gpuVa;
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps);
if (overlapsCount != 0)
_virtualRanges.Lock.EnterWriteLock();
result = _virtualRanges.FindOverlapsNonOverlappingAsSpan(gpuVa, size, out ReadOnlySpan<RangeItem<VirtualRange>> overlaps);
if (overlaps.Length != 0)
{
// The virtual range already exists. We just need to check if our range fits inside
// the existing one, and if not, we must extend the existing one.
ulong endAddress = gpuVa + size;
VirtualRange overlap0 = overlaps[0];
VirtualRange overlap0 = overlaps[0].Value;
if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress)
{
for (int index = 0; index < overlapsCount; index++)
{
VirtualRange virtualRange = overlaps[index];
gpuVa = Math.Min(gpuVa, virtualRange.Address);
endAddress = Math.Max(endAddress, virtualRange.EndAddress);
_virtualRanges.Remove(virtualRange);
}
gpuVa = Math.Min(gpuVa, overlaps[0].Address);
endAddress = Math.Max(endAddress, overlaps[^1].EndAddress);
_virtualRanges.RemoveRange(result);
ulong newSize = endAddress - gpuVa;
MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize);
@ -170,6 +165,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_virtualRanges.Add(virtualRange);
}
_virtualRanges.Lock.ExitWriteLock();
ShrinkOverlapsBufferIfNeeded();

15
src/Ryujinx.Memory/Range/NonOverlappingRangeList.cs
View file

@ -19,16 +19,18 @@ namespace Ryujinx.Memory.Range
/// <param name="address">Start address of the search region</param>
/// <param name="size">Size of the search region</param>
/// <param name="factory">Factory for creating new ranges</param>
public void GetOrAddRegions(List<T> list, ulong address, ulong size, Func<ulong, ulong, T> factory)
public void GetOrAddRegions(out List<T> list, ulong address, ulong size, Func<ulong, ulong, T> factory)
{
// (regarding the specific case this generalized function is used for)
// A new region may be split into multiple parts if multiple virtual regions have mapped to it.
// For instance, while a virtual mapping could cover 0-2 in physical space, the space 0-1 may have already been reserved...
// So we need to return both the split 0-1 and 1-2 ranges.
T[] results = new T[1];
int count = FindOverlapsNonOverlapping(address, size, ref results);
RangeItem<T>[] results = new RangeItem<T>[1];
int count = FindOverlapsNonOverlapping(address, size, ref results).Count;
list = new List<T>(count);
Lock.EnterWriteLock();
if (count == 0)
{
// The region is fully unmapped. Create and add it to the range list.
@ -43,11 +45,12 @@ namespace Ryujinx.Memory.Range
for (int i = 0; i < count; i++)
{
T region = results[i];
T region = results[i].Value;
if (count == 1 && region.Address == address && region.Size == size)
{
// Exact match, no splitting required.
list.Add(region);
Lock.ExitWriteLock();
return;
}
@ -85,6 +88,8 @@ namespace Ryujinx.Memory.Range
Add(fillRegion);
}
}
Lock.ExitWriteLock();
}
/// <summary>

480
src/Ryujinx.Memory/Range/RangeList.cs
View file

@ -2,44 +2,63 @@ using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
namespace Ryujinx.Memory.Range
{
public readonly struct RangeItem<TValue>(TValue value) where TValue : IRange
{
public readonly ulong Address = value.Address;
public readonly ulong EndAddress = value.Address + value.Size;
public readonly TValue Value = value;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool OverlapsWith(ulong address, ulong endAddress)
{
return Address < endAddress && address < EndAddress;
}
}
/// <summary>
/// Result of an Overlaps Finder function.
/// </summary>
/// <remarks>
/// startIndex is inclusive.
/// endIndex is exclusive.
/// </remarks>
public readonly struct OverlapResult
{
public readonly int StartIndex = -1;
public readonly int EndIndex = -1;
public int Count => EndIndex - StartIndex;
public OverlapResult(int startIndex, int endIndex)
{
this.StartIndex = startIndex;
this.EndIndex = endIndex;
}
}
/// <summary>
/// Sorted list of ranges that supports binary search.
/// </summary>
/// <typeparam name="T">Type of the range.</typeparam>
public class RangeList<T> : IEnumerable<T> where T : IRange
{
private readonly struct RangeItem<TValue> where TValue : IRange
{
public readonly ulong Address;
public readonly ulong EndAddress;
public readonly TValue Value;
public RangeItem(TValue value)
{
Value = value;
Address = value.Address;
EndAddress = value.Address + value.Size;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool OverlapsWith(ulong address, ulong endAddress)
{
return Address < endAddress && address < EndAddress;
}
}
private const int BackingInitialSize = 1024;
private const int ArrayGrowthSize = 32;
private RangeItem<T>[] _items;
private readonly int _backingGrowthSize;
public int Count { get; protected set; }
public readonly ReaderWriterLockSlim Lock = new();
private const int QuickAccessLength = 8;
private int _offset;
private int _count;
private RangeItem<T>[] _quickAccess = new RangeItem<T>[QuickAccessLength];
/// <summary>
/// Creates a new range list.
@ -72,22 +91,21 @@ namespace Ryujinx.Memory.Range
/// </summary>
/// <param name="item">The item to be updated</param>
/// <returns>True if the item was located and updated, false otherwise</returns>
public bool Update(T item)
protected bool Update(T item)
{
int index = BinarySearch(item.Address);
if (index >= 0)
{
while (index > 0 && _items[index - 1].Address == item.Address)
{
index--;
}
while (index < Count)
{
if (_items[index].Value.Equals(item))
{
_items[index] = new RangeItem<T>(item);
_quickAccess = new RangeItem<T>[QuickAccessLength];
_count = 0;
_offset = 0;
return true;
}
@ -129,12 +147,34 @@ namespace Ryujinx.Memory.Range
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveAt(int index)
protected void RemoveAt(int index)
{
if (index < --Count)
{
Array.Copy(_items, index + 1, _items, index, Count - index);
}
_quickAccess = new RangeItem<T>[QuickAccessLength];
_count = 0;
_offset = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void RemoveRange(OverlapResult overlapResult)
{
if (overlapResult.EndIndex < Count)
{
Array.Copy(_items, overlapResult.EndIndex, _items, overlapResult.StartIndex, Count - overlapResult.EndIndex);
Count -= overlapResult.Count;
}
else if (overlapResult.EndIndex == Count)
{
Count = overlapResult.StartIndex;
}
_quickAccess = new RangeItem<T>[QuickAccessLength];
_count = 0;
_offset = 0;
}
/// <summary>
@ -148,11 +188,6 @@ namespace Ryujinx.Memory.Range
if (index >= 0)
{
while (index > 0 && _items[index - 1].Address == item.Address)
{
index--;
}
while (index < Count)
{
if (_items[index].Value.Equals(item))
@ -175,67 +210,39 @@ namespace Ryujinx.Memory.Range
}
/// <summary>
/// Updates an item's end address.
/// </summary>
/// <param name="item">The item to be updated</param>
public void UpdateEndAddress(T item)
{
int index = BinarySearch(item.Address);
if (index >= 0)
{
while (index > 0 && _items[index - 1].Address == item.Address)
{
index--;
}
while (index < Count)
{
if (_items[index].Value.Equals(item))
{
_items[index] = new RangeItem<T>(item);
return;
}
if (_items[index].Address > item.Address)
{
break;
}
index++;
}
}
}
/// <summary>
/// Gets the first item on the list overlapping in memory with the specified item.
/// Gets an item on the list overlapping the specified memory range.
/// </summary>
/// <remarks>
/// Despite the name, this has no ordering guarantees of the returned item.
/// It only ensures that the item returned overlaps the specified item.
/// </remarks>
/// <param name="item">Item to check for overlaps</param>
/// <returns>The overlapping item, or the default value for the type if none found</returns>
public T FindFirstOverlap(T item)
{
return FindFirstOverlap(item.Address, item.Size);
}
/// <summary>
/// Gets the first item on the list overlapping the specified memory range.
/// </summary>
/// <remarks>
/// Despite the name, this has no ordering guarantees of the returned item.
/// This has no ordering guarantees of the returned item.
/// It only ensures that the item returned overlaps the specified memory range.
/// </remarks>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param>
/// <returns>The overlapping item, or the default value for the type if none found</returns>
public T FindFirstOverlap(ulong address, ulong size)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T FindOverlapFast(ulong address, ulong size)
{
for (int i = 0; i < _quickAccess.Length; i++)
{
ref RangeItem<T> item = ref _quickAccess[(i + _offset) % _quickAccess.Length];
if (item.OverlapsWith(address, address + size))
{
return item.Value;
}
}
int index = BinarySearch(address, address + size);
if (_count < _quickAccess.Length)
{
_quickAccess[_count++] = _items[index];
}
else
{
_quickAccess[_offset++ % _quickAccess.Length] = _items[index];
}
if (index < 0)
{
return default;
@ -243,68 +250,66 @@ namespace Ryujinx.Memory.Range
return _items[index].Value;
}
/// <summary>
/// Gets all items overlapping with the specified item in memory.
/// </summary>
/// <param name="item">Item to check for overlaps</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of overlapping items found</returns>
public int FindOverlaps(T item, ref T[] output)
{
return FindOverlaps(item.Address, item.Size, ref output);
}
/// <summary>
/// Gets all items on the list overlapping the specified memory range.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of overlapping items found</returns>
public int FindOverlaps(ulong address, ulong size, ref T[] output)
/// <returns>Range information of overlapping items found</returns>
public OverlapResult FindOverlaps(ulong address, ulong size, ref RangeItem<T>[] output)
{
int outputIndex = 0;
ulong endAddress = address + size;
int startIndex = BinarySearch(address);
if (startIndex < 0)
startIndex = ~startIndex;
int endIndex = -1;
for (int i = 0; i < Count; i++)
for (int i = startIndex; i < Count; i++)
{
ref RangeItem<T> item = ref _items[i];
if (item.Address >= endAddress)
{
endIndex = i;
break;
}
if (item.OverlapsWith(address, endAddress))
{
if (outputIndex == output.Length)
{
Array.Resize(ref output, outputIndex + ArrayGrowthSize);
}
output[outputIndex++] = item.Value;
outputIndex++;
}
}
return outputIndex;
}
if (endIndex == -1 && outputIndex > 0)
{
endIndex = Count;
}
/// <summary>
/// Gets all items overlapping with the specified item in memory.
/// </summary>
/// <remarks>
/// This method only returns correct results if none of the items on the list overlaps with
/// each other. If that is not the case, this method should not be used.
/// This method is faster than the regular method to find all overlaps.
/// </remarks>
/// <param name="item">Item to check for overlaps</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of overlapping items found</returns>
public int FindOverlapsNonOverlapping(T item, ref T[] output)
{
return FindOverlapsNonOverlapping(item.Address, item.Size, ref output);
if (outputIndex > 0 && outputIndex == endIndex - startIndex)
{
Array.Resize(ref output, outputIndex);
Array.Copy(_items, endIndex - outputIndex, output, 0, outputIndex);
return new OverlapResult(startIndex, endIndex);
}
else if (outputIndex > 0)
{
Array.Resize(ref output, outputIndex);
int arrIndex = 0;
for (int i = startIndex; i < endIndex; i++)
{
output[arrIndex++] = _items[i];
}
return new OverlapResult(endIndex - outputIndex, endIndex);
}
return new OverlapResult();
}
/// <summary>
@ -318,77 +323,68 @@ namespace Ryujinx.Memory.Range
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of overlapping items found</returns>
public int FindOverlapsNonOverlapping(ulong address, ulong size, ref T[] output)
/// <returns>Range information of overlapping items found</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size, ref RangeItem<T>[] output)
{
// This is a bit faster than FindOverlaps, but only works
// when none of the items on the list overlaps with each other.
int outputIndex = 0;
ulong endAddress = address + size;
int index = BinarySearch(address, endAddress);
(int index, int endIndex) = BinarySearchEdges(address, endAddress);
if (index >= 0)
{
while (index > 0 && _items[index - 1].OverlapsWith(address, endAddress))
{
index--;
}
do
{
if (outputIndex == output.Length)
{
Array.Resize(ref output, outputIndex + ArrayGrowthSize);
}
output[outputIndex++] = _items[index++].Value;
}
while (index < Count && _items[index].OverlapsWith(address, endAddress));
Array.Resize(ref output, endIndex - index);
Array.Copy(_items, index, output, 0, endIndex - index);
}
return outputIndex;
return new OverlapResult(index, endIndex);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public OverlapResult FindOverlapsNonOverlappingAsSpan(ulong address, ulong size, out ReadOnlySpan<RangeItem<T>> span)
{
// This is a bit faster than FindOverlaps, but only works
// when none of the items on the list overlaps with each other.
ulong endAddress = address + size;
(int index, int endIndex) = BinarySearchEdges(address, endAddress);
if (index >= 0)
{
span = new ReadOnlySpan<RangeItem<T>>(_items, index, endIndex - index);
return new OverlapResult(index, endIndex);
}
span = ReadOnlySpan<RangeItem<T>>.Empty;
return new OverlapResult(index, endIndex);
}
/// <summary>
/// Gets all items on the list with the specified memory address.
/// Gets the range of all items on the list overlapping the specified memory range.
/// </summary>
/// <param name="address">Address to find</param>
/// <param name="output">Output array where matches will be written. It is automatically resized to fit the results</param>
/// <returns>The number of matches found</returns>
public int FindOverlaps(ulong address, ref T[] output)
/// <remarks>
/// This method only returns correct results if none of the items on the list overlaps with
/// each other. If that is not the case, this method should not be used.
/// This method is faster than the regular method to find all overlaps.
/// </remarks>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param>
/// <returns>Range information of overlapping items found</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public OverlapResult FindOverlapsNonOverlapping(ulong address, ulong size)
{
int index = BinarySearch(address);
// This is a bit faster than FindOverlaps, but only works
// when none of the items on the list overlaps with each other.
int outputIndex = 0;
ulong endAddress = address + size;
if (index >= 0)
{
while (index > 0 && _items[index - 1].Address == address)
{
index--;
}
(int index, int endIndex) = BinarySearchEdges(address, endAddress);
while (index < Count)
{
ref RangeItem<T> overlap = ref _items[index++];
if (overlap.Address != address)
{
break;
}
if (outputIndex == output.Length)
{
Array.Resize(ref output, outputIndex + ArrayGrowthSize);
}
output[outputIndex++] = overlap.Value;
}
}
return outputIndex;
return new OverlapResult(index, endIndex);
}
/// <summary>
@ -396,6 +392,7 @@ namespace Ryujinx.Memory.Range
/// </summary>
/// <param name="address">Address to find</param>
/// <returns>List index of the item, or complement index of nearest item with lower value on the list</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int BinarySearch(ulong address)
{
int left = 0;
@ -426,13 +423,14 @@ namespace Ryujinx.Memory.Range
return ~left;
}
/// <summary>
/// Performs binary search for items overlapping a given memory range.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="endAddress">End address of the range</param>
/// <returns>List index of the item, or complement index of nearest item with lower value on the list</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int BinarySearch(ulong address, ulong endAddress)
{
int left = 0;
@ -464,6 +462,136 @@ namespace Ryujinx.Memory.Range
return ~left;
}
/// <summary>
/// Performs binary search for items overlapping a given memory range.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="endAddress">End address of the range</param>
/// <returns>Range information (inclusive, exclusive) of items that overlaps, or complement index of nearest item with lower value on the list</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private (int, int) BinarySearchEdges(ulong address, ulong endAddress)
{
if (Count == 0)
return (~0, ~0);
if (Count == 1)
{
ref RangeItem<T> item = ref _items[0];
if (item.OverlapsWith(address, endAddress))
{
return (0, 1);
}
if (address < item.Address)
{
return (~0, ~0);
}
else
{
return (~1, ~1);
}
}
int left = 0;
int right = Count - 1;
int leftEdge = -1;
int rightEdgeMatch = -1;
int rightEdgeNoMatch = -1;
while (left <= right)
{
int range = right - left;
int middle = left + (range >> 1);
ref RangeItem<T> item = ref _items[middle];
bool match = item.OverlapsWith(address, endAddress);
if (range == 0)
{
if (match)
{
leftEdge = middle;
break;
}
else if (address < item.Address)
{
return (~right, ~right);
}
else
{
return (~(right + 1), ~(right + 1));
}
}
if (match)
{
right = middle;
if (rightEdgeMatch == -1)
rightEdgeMatch = middle;
}
else if (address < item.Address)
{
right = middle - 1;
rightEdgeNoMatch = middle;
}
else
{
left = middle + 1;
}
}
if (left > right)
{
return (~left, ~left);
}
if (rightEdgeMatch == -1)
{
return (leftEdge, leftEdge + 1);
}
left = rightEdgeMatch;
right = rightEdgeNoMatch > 0 ? rightEdgeNoMatch : Count - 1;
while (left <= right)
{
int range = right - left;
int middle = right - (range >> 1);
ref RangeItem<T> item = ref _items[middle];
bool match = item.OverlapsWith(address, endAddress);
if (range == 0)
{
if (match)
return (leftEdge, middle + 1);
else
return (leftEdge, middle);
}
if (match)
{
left = middle;
}
else if (address < item.Address)
{
right = middle - 1;
}
else
{
left = middle + 1;
}
}
return (leftEdge, right + 1);
}
public IEnumerator<T> GetEnumerator()
{
for (int i = 0; i < Count; i++)

55
src/Ryujinx.Memory/Tracking/MemoryTracking.cs
View file

@ -1,5 +1,6 @@
using Ryujinx.Common.Pools;
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
namespace Ryujinx.Memory.Tracking
@ -76,17 +77,15 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock)
{
ref VirtualRegion[] overlaps = ref ThreadStaticArray<VirtualRegion>.Get();
for (int type = 0; type < 2; type++)
{
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps);
for (int i = 0; i < count; i++)
regions.Lock.EnterReadLock();
regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan<RangeItem<VirtualRegion>> overlaps);
for (int i = 0; i < overlaps.Length; i++)
{
VirtualRegion region = overlaps[i];
VirtualRegion region = overlaps[i].Value;
// If the region has been fully remapped, signal that it has been mapped again.
bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size);
@ -97,6 +96,7 @@ namespace Ryujinx.Memory.Tracking
region.UpdateProtection();
}
regions.Lock.ExitReadLock();
}
}
}
@ -114,20 +114,19 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock)
{
ref VirtualRegion[] overlaps = ref ThreadStaticArray<VirtualRegion>.Get();
for (int type = 0; type < 2; type++)
{
NonOverlappingRangeList<VirtualRegion> regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
regions.Lock.EnterReadLock();
regions.FindOverlapsNonOverlappingAsSpan(va, size, out ReadOnlySpan<RangeItem<VirtualRegion>> overlaps);
int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps);
for (int i = 0; i < count; i++)
for (int i = 0; i < overlaps.Length; i++)
{
VirtualRegion region = overlaps[i];
VirtualRegion region = overlaps[i].Value;
region.SignalMappingChanged(false);
}
regions.Lock.ExitReadLock();
}
}
}
@ -165,10 +164,11 @@ namespace Ryujinx.Memory.Tracking
/// <returns>A list of virtual regions within the given range</returns>
internal List<VirtualRegion> GetVirtualRegionsForHandle(ulong va, ulong size, bool guest)
{
List<VirtualRegion> result = [];
NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions;
regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest));
regions.Lock.EnterUpgradeableReadLock();
regions.GetOrAddRegions(out List<VirtualRegion> result, va, size, (va, size) => new VirtualRegion(this, va, size, guest));
regions.Lock.ExitUpgradeableReadLock();
return result;
}
@ -296,25 +296,26 @@ namespace Ryujinx.Memory.Tracking
lock (TrackingLock)
{
ref VirtualRegion[] overlaps = ref ThreadStaticArray<VirtualRegion>.Get();
NonOverlappingRangeList<VirtualRegion> regions = guest ? _guestVirtualRegions : _virtualRegions;
ref RangeItem<VirtualRegion>[] overlaps = ref ThreadStaticArray<RangeItem<VirtualRegion>>.Get();
// We use the non-span method here because keeping the lock will cause a deadlock.
regions.Lock.EnterReadLock();
OverlapResult result = regions.FindOverlapsNonOverlapping(address, size, ref overlaps);
regions.Lock.ExitReadLock();
int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps);
if (count == 0 && !precise)
if (overlaps.Length == 0 && !precise)
{
if (_memoryManager.IsRangeMapped(address, size))
{
// TODO: There is currently the possibility that a page can be protected after its virtual region is removed.
// This code handles that case when it happens, but it would be better to find out how this happens.
_memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest);
return true; // This memory _should_ be mapped, so we need to try again.
}
else
{
shouldThrow = true;
}
shouldThrow = true;
}
else
{
@ -324,9 +325,9 @@ namespace Ryujinx.Memory.Tracking
size += (ulong)_pageSize;
}
for (int i = 0; i < count; i++)
for (int i = 0; i < result.Count; i++)
{
VirtualRegion region = overlaps[i];
VirtualRegion region = overlaps[i].Value;
if (precise)
{