Texture/Vertex/Index data cache (#132)

* Initial implementation of the texture cache

* Cache vertex and index data aswell, some cleanup

* Improve handling of the cache by storing cached ranges on a list for each page

* Delete old data from the caches automatically, ensure that the cache is cleaned when the mapping/size changes, and some general cleanup

Ryujinx.Core/Gpu/NvGpu.cs

@@ -3,7 +3,7 @@ using System.Threading;
 
 namespace Ryujinx.Core.Gpu
 {
-    public class NvGpu
+    class NvGpu
     {
         public IGalRenderer Renderer { get; private set; }
 

Ryujinx.Core/Gpu/NvGpuBufferType.cs

@@ -0,0 +1,9 @@
+namespace Ryujinx.Core.Gpu
+{
+    enum NvGpuBufferType
+    {
+        Index,
+        Vertex,
+        Texture
+    }
+}

Ryujinx.Core/Gpu/NvGpuEngine2d.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 
 namespace Ryujinx.Core.Gpu
 {
-    public class NvGpuEngine2d : INvGpuEngine
+    class NvGpuEngine2d : INvGpuEngine
     {
         private enum CopyOperation
         {

Ryujinx.Core/Gpu/NvGpuEngine3d.cs

@@ -4,7 +4,7 @@ using System.Collections.Generic;
 
 namespace Ryujinx.Core.Gpu
 {
-    public class NvGpuEngine3d : INvGpuEngine
+    class NvGpuEngine3d : INvGpuEngine
     {
         public int[] Registers { get; private set; }
 
@@ -261,6 +261,8 @@ namespace Ryujinx.Core.Gpu
 
             long TextureAddress = Vmm.ReadInt64(TicPosition + 4) & 0xffffffffffff;
 
+            long Tag = TextureAddress;
+
             TextureAddress = Vmm.GetPhysicalAddress(TextureAddress);
 
             if (IsFrameBufferPosition(TextureAddress))
@@ -273,10 +275,25 @@ namespace Ryujinx.Core.Gpu
             }
             else
             {
-                GalTexture Texture = TextureFactory.MakeTexture(Gpu, Vmm, TicPosition);
+                GalTexture NewTexture = TextureFactory.MakeTexture(Vmm, TicPosition);
 
-                Gpu.Renderer.SetTextureAndSampler(TexIndex, Texture, Sampler);
-                Gpu.Renderer.BindTexture(TexIndex);
+                long Size = (uint)TextureHelper.GetTextureSize(NewTexture);
+
+                if (Gpu.Renderer.TryGetCachedTexture(Tag, Size, out GalTexture Texture))
+                {
+                    if (NewTexture.Equals(Texture) && !Vmm.IsRegionModified(Tag, Size, NvGpuBufferType.Texture))
+                    {
+                        Gpu.Renderer.BindTexture(Tag, TexIndex);
+
+                        return;
+                    }
+                }
+
+                byte[] Data = TextureFactory.GetTextureData(Vmm, TicPosition);
+
+                Gpu.Renderer.SetTextureAndSampler(Tag, Data, NewTexture, Sampler);
+
+                Gpu.Renderer.BindTexture(Tag, TexIndex);
             }
         }
 
@@ -330,11 +347,18 @@ namespace Ryujinx.Core.Gpu
 
             if (IndexSize != 0)
             {
-                int BufferSize = IndexCount * IndexSize;
+                int IbSize = IndexCount * IndexSize;
 
-                byte[] Data = Vmm.ReadBytes(IndexPosition, BufferSize);
+                bool IboCached = Gpu.Renderer.IsIboCached(IndexPosition, (uint)IbSize);
 
-                Gpu.Renderer.SetIndexArray(Data, IndexFormat);
+                if (!IboCached || Vmm.IsRegionModified(IndexPosition, (uint)IbSize, NvGpuBufferType.Index))
+                {
+                    byte[] Data = Vmm.ReadBytes(IndexPosition, (uint)IbSize);
+
+                    Gpu.Renderer.CreateIbo(IndexPosition, Data);
+                }
+
+                Gpu.Renderer.SetIndexArray(IndexPosition, IbSize, IndexFormat);
             }
 
             List<GalVertexAttrib>[] Attribs = new List<GalVertexAttrib>[32];
@@ -359,10 +383,17 @@ namespace Ryujinx.Core.Gpu
                                          ((Packed >> 31) & 0x1) != 0));
             }
 
+            int VertexFirst = ReadRegister(NvGpuEngine3dReg.VertexArrayFirst);
+            int VertexCount = ReadRegister(NvGpuEngine3dReg.VertexArrayCount);
+
+            int PrimCtrl = ReadRegister(NvGpuEngine3dReg.VertexBeginGl);
+
             for (int Index = 0; Index < 32; Index++)
             {
-                int VertexFirst = ReadRegister(NvGpuEngine3dReg.VertexArrayFirst);
-                int VertexCount = ReadRegister(NvGpuEngine3dReg.VertexArrayCount);
+                if (Attribs[Index] == null)
+                {
+                    continue;
+                }
 
                 int Control = ReadRegister(NvGpuEngine3dReg.VertexArrayNControl + Index * 4);
 
@@ -378,39 +409,38 @@ namespace Ryujinx.Core.Gpu
 
                 int Stride = Control & 0xfff;
 
-                long Size = 0;
+                long VbSize = 0;
 
                 if (IndexCount != 0)
                 {
-                    Size = (VertexEndPos - VertexPosition) + 1;
+                    VbSize = (VertexEndPos - VertexPosition) + 1;
                 }
                 else
                 {
-                    Size = VertexCount;
+                    VbSize = VertexCount * Stride;
                 }
 
-                //TODO: Support cases where the Stride is 0.
-                //In this case, we need to use the size of the attribute.
-                Size *= Stride;
+                bool VboCached = Gpu.Renderer.IsVboCached(VertexPosition, VbSize);
 
-                byte[] Data = Vmm.ReadBytes(VertexPosition, Size);
-
-                GalVertexAttrib[] AttribArray = Attribs[Index]?.ToArray() ?? new GalVertexAttrib[0];
-
-                Gpu.Renderer.SetVertexArray(Index, Stride, Data, AttribArray);
-
-                int PrimCtrl = ReadRegister(NvGpuEngine3dReg.VertexBeginGl);
-
-                GalPrimitiveType PrimType = (GalPrimitiveType)(PrimCtrl & 0xffff);
-
-                if (IndexCount != 0)
+                if (!VboCached || Vmm.IsRegionModified(VertexPosition, VbSize, NvGpuBufferType.Vertex))
                 {
-                    Gpu.Renderer.DrawElements(Index, IndexFirst, PrimType);
-                }
-                else
-                {
-                    Gpu.Renderer.DrawArrays(Index, VertexFirst, VertexCount, PrimType);
+                    byte[] Data = Vmm.ReadBytes(VertexPosition, VbSize);
+
+                    Gpu.Renderer.CreateVbo(VertexPosition, Data);
                 }
+
+                Gpu.Renderer.SetVertexArray(Index, Stride, VertexPosition, Attribs[Index].ToArray());
+            }
+
+            GalPrimitiveType PrimType = (GalPrimitiveType)(PrimCtrl & 0xffff);
+
+            if (IndexCount != 0)
+            {
+                Gpu.Renderer.DrawElements(IndexPosition, IndexFirst, PrimType);
+            }
+            else
+            {
+                Gpu.Renderer.DrawArrays(VertexFirst, VertexCount, PrimType);
             }
         }
 

Ryujinx.Core/Gpu/NvGpuFifo.cs

@@ -2,7 +2,7 @@ using System.Collections.Concurrent;
 
 namespace Ryujinx.Core.Gpu
 {
-    public class NvGpuFifo
+    class NvGpuFifo
     {
         private const int MacrosCount    = 0x80;
         private const int MacroIndexMask = MacrosCount - 1;

Ryujinx.Core/Gpu/NvGpuPBEntry.cs

@@ -3,7 +3,7 @@ using System.Collections.ObjectModel;
 
 namespace Ryujinx.Core.Gpu
 {
-    public struct NvGpuPBEntry
+    struct NvGpuPBEntry
     {
         public int Method { get; private set; }
 

Ryujinx.Core/Gpu/NvGpuPushBuffer.cs

@@ -3,7 +3,7 @@ using System.IO;
 
 namespace Ryujinx.Core.Gpu
 {
-    public static class NvGpuPushBuffer
+    static class NvGpuPushBuffer
     {
         private enum SubmissionMode
         {

Ryujinx.Core/Gpu/NvGpuVmm.cs

@@ -4,24 +4,24 @@ using System.Collections.Concurrent;
 
 namespace Ryujinx.Core.Gpu
 {
-    public class NvGpuVmm : IAMemory, IGalMemory
+    class NvGpuVmm : IAMemory, IGalMemory
     {
         public const long AddrSize = 1L << 40;
 
-        private const int  PTLvl0Bits = 14;
-        private const int  PTLvl1Bits = 14;
-        private const int  PTPageBits = 12;
+        private const int PTLvl0Bits = 14;
+        private const int PTLvl1Bits = 14;
+        private const int PTPageBits = 12;
 
-        private const int  PTLvl0Size = 1 << PTLvl0Bits;
-        private const int  PTLvl1Size = 1 << PTLvl1Bits;
-        public  const int  PageSize   = 1 << PTPageBits;
+        private const int PTLvl0Size = 1 << PTLvl0Bits;
+        private const int PTLvl1Size = 1 << PTLvl1Bits;
+        public  const int PageSize   = 1 << PTPageBits;
 
-        private const int  PTLvl0Mask = PTLvl0Size - 1;
-        private const int  PTLvl1Mask = PTLvl1Size - 1;
-        public  const int  PageMask   = PageSize   - 1;
+        private const int PTLvl0Mask = PTLvl0Size - 1;
+        private const int PTLvl1Mask = PTLvl1Size - 1;
+        public  const int PageMask   = PageSize   - 1;
 
-        private const int  PTLvl0Bit = PTPageBits + PTLvl1Bits;
-        private const int  PTLvl1Bit = PTPageBits;
+        private const int PTLvl0Bit = PTPageBits + PTLvl1Bits;
+        private const int PTLvl1Bit = PTPageBits;
 
         public AMemory Memory { get; private set; }
 
@@ -37,6 +37,8 @@ namespace Ryujinx.Core.Gpu
 
         private ConcurrentDictionary<long, MappedMemory> Maps;
 
+        private NvGpuVmmCache Cache;
+
         private const long PteUnmapped = -1;
         private const long PteReserved = -2;
 
@@ -48,6 +50,8 @@ namespace Ryujinx.Core.Gpu
 
             Maps = new ConcurrentDictionary<long, MappedMemory>();
 
+            Cache = new NvGpuVmmCache();
+
             PageTable = new long[PTLvl0Size][];
         }
 
@@ -270,6 +274,13 @@ namespace Ryujinx.Core.Gpu
             PageTable[L0][L1] = TgtAddr;
         }
 
+        public bool IsRegionModified(long Position, long Size, NvGpuBufferType BufferType)
+        {
+            long PA = GetPhysicalAddress(Position);
+
+            return Cache.IsRegionModified(Memory, BufferType, Position, PA, Size);
+        }
+
         public byte ReadByte(long Position)
         {
             Position = GetPhysicalAddress(Position);

Ryujinx.Core/Gpu/NvGpuVmmCache.cs

@@ -0,0 +1,209 @@
+using ChocolArm64.Memory;
+using System;
+using System.Collections.Generic;
+
+namespace Ryujinx.Core.Gpu
+{
+    class NvGpuVmmCache
+    {
+        private const int MaxCpCount     = 10000;
+        private const int MaxCpTimeDelta = 60000;
+
+        private class CachedPage
+        {
+            private List<(long Start, long End)> Regions;
+
+            public LinkedListNode<long> Node { get; set; }
+
+            public int Count => Regions.Count;
+
+            public int Timestamp { get; private set; }
+
+            public long PABase { get; private set; }
+
+            public NvGpuBufferType BufferType { get; private set; }
+
+            public CachedPage(long PABase, NvGpuBufferType BufferType)
+            {
+                this.PABase     = PABase;
+                this.BufferType = BufferType;
+
+                Regions = new List<(long, long)>();
+            }
+
+            public bool AddRange(long Start, long End)
+            {
+                for (int Index = 0; Index < Regions.Count; Index++)
+                {
+                    (long RgStart, long RgEnd) = Regions[Index];
+
+                    if (Start >= RgStart && End <= RgEnd)
+                    {
+                        return false;
+                    }
+
+                    if (Start <= RgEnd && RgStart <= End)
+                    {
+                        long MinStart = Math.Min(RgStart, Start);
+                        long MaxEnd   = Math.Max(RgEnd,   End);
+
+                        Regions[Index] = (MinStart, MaxEnd);
+
+                        Timestamp = Environment.TickCount;
+
+                        return true;
+                    }
+                }
+
+                Regions.Add((Start, End));
+
+                Timestamp = Environment.TickCount;
+
+                return true;
+            }
+        }
+
+        private Dictionary<long, CachedPage> Cache;
+
+        private LinkedList<long> SortedCache;
+
+        private int CpCount;
+
+        public NvGpuVmmCache()
+        {
+            Cache = new Dictionary<long, CachedPage>();
+
+            SortedCache = new LinkedList<long>();
+        }
+
+        public bool IsRegionModified(
+            AMemory         Memory,
+            NvGpuBufferType BufferType,
+            long            VA,
+            long            PA,
+            long            Size)
+        {
+            ClearCachedPagesIfNeeded();
+
+            long PageSize = Memory.GetHostPageSize();
+
+            long Mask = PageSize - 1;
+
+            long VAEnd = VA + Size;
+            long PAEnd = PA + Size;
+
+            bool RegMod = false;
+
+            while (VA < VAEnd)
+            {
+                long Key    = VA & ~Mask;
+                long PABase = PA & ~Mask;
+
+                long VAPgEnd = Math.Min((VA + PageSize) & ~Mask, VAEnd);
+                long PAPgEnd = Math.Min((PA + PageSize) & ~Mask, PAEnd);
+
+                bool IsCached = Cache.TryGetValue(Key, out CachedPage Cp);
+
+                bool PgReset = false;
+
+                if (!IsCached)
+                {
+                    Cp = new CachedPage(PABase, BufferType);
+
+                    Cache.Add(Key, Cp);
+                }
+                else
+                {
+                    CpCount -= Cp.Count;
+
+                    SortedCache.Remove(Cp.Node);
+
+                    if (Cp.PABase     != PABase ||
+                        Cp.BufferType != BufferType)
+                    {
+                        PgReset = true;
+                    }
+                }
+
+                PgReset |= Memory.IsRegionModified(PA, PAPgEnd - PA) && IsCached;
+
+                if (PgReset)
+                {
+                    Cp = new CachedPage(PABase, BufferType);
+
+                    Cache[Key] = Cp;
+                }
+
+                Cp.Node = SortedCache.AddLast(Key);
+
+                RegMod |= Cp.AddRange(VA, VAPgEnd);
+
+                CpCount += Cp.Count;
+
+                VA = VAPgEnd;
+                PA = PAPgEnd;
+            }
+
+            return RegMod;
+        }
+
+        private void ClearCachedPagesIfNeeded()
+        {
+            if (CpCount <= MaxCpCount)
+            {
+                return;
+            }
+
+            int Timestamp = Environment.TickCount;
+
+            int TimeDelta;
+
+            do
+            {
+                if (!TryPopOldestCachedPageKey(Timestamp, out long Key))
+                {
+                    break;
+                }
+
+                CachedPage Cp = Cache[Key];
+
+                Cache.Remove(Key);
+
+                CpCount -= Cp.Count;
+
+                TimeDelta = RingDelta(Cp.Timestamp, Timestamp);
+            }
+            while (CpCount > (MaxCpCount >> 1) || (uint)TimeDelta > (uint)MaxCpTimeDelta);
+        }
+
+        private bool TryPopOldestCachedPageKey(int Timestamp, out long Key)
+        {
+            LinkedListNode<long> Node = SortedCache.First;
+
+            if (Node == null)
+            {
+                Key = 0;
+
+                return false;
+            }
+
+            SortedCache.Remove(Node);
+
+            Key = Node.Value;
+
+            return true;
+        }
+
+        private int RingDelta(int Old, int New)
+        {
+            if ((uint)New < (uint)Old)
+            {
+                return New + (~Old + 1);
+            }
+            else
+            {
+                return New - Old;
+            }
+        }
+    }
+}

Ryujinx.Core/Gpu/Texture.cs

@@ -2,7 +2,7 @@ using Ryujinx.Graphics.Gal;
 
 namespace Ryujinx.Core.Gpu
 {
-    public struct Texture
+    struct Texture
     {
         public long Position { get; private set; }
 

Ryujinx.Core/Gpu/TextureFactory.cs

@@ -5,7 +5,7 @@ namespace Ryujinx.Core.Gpu
 {
     static class TextureFactory
     {
-        public static GalTexture MakeTexture(NvGpu Gpu, NvGpuVmm Vmm, long TicPosition)
+        public static GalTexture MakeTexture(NvGpuVmm Vmm, long TicPosition)
         {
             int[] Tic = ReadWords(Vmm, TicPosition, 8);
 
@@ -16,6 +16,25 @@ namespace Ryujinx.Core.Gpu
             GalTextureSource ZSource = (GalTextureSource)((Tic[0] >> 25) & 7);
             GalTextureSource WSource = (GalTextureSource)((Tic[0] >> 28) & 7);
 
+            int Width  = (Tic[4] & 0xffff) + 1;
+            int Height = (Tic[5] & 0xffff) + 1;
+
+            return new GalTexture(
+                Width,
+                Height,
+                Format,
+                XSource,
+                YSource,
+                ZSource,
+                WSource);
+        }
+
+        public static byte[] GetTextureData(NvGpuVmm Vmm, long TicPosition)
+        {
+            int[] Tic = ReadWords(Vmm, TicPosition, 8);
+
+            GalTextureFormat Format = (GalTextureFormat)(Tic[0] & 0x7f);
+
             long TextureAddress = (uint)Tic[1];
 
             TextureAddress |= (long)((ushort)Tic[2]) << 32;
@@ -40,17 +59,7 @@ namespace Ryujinx.Core.Gpu
                 Swizzle,
                 Format);
 
-            byte[] Data = TextureReader.Read(Vmm, Texture);
-
-            return new GalTexture(
-                Data,
-                Width,
-                Height,
-                Format,
-                XSource,
-                YSource,
-                ZSource,
-                WSource);
+            return TextureReader.Read(Vmm, Texture);
         }
 
         public static GalTextureSampler MakeSampler(NvGpu Gpu, NvGpuVmm Vmm, long TscPosition)

Ryujinx.Core/Gpu/TextureHelper.cs

@@ -1,4 +1,5 @@
 using ChocolArm64.Memory;
+using Ryujinx.Graphics.Gal;
 using System;
 
 namespace Ryujinx.Core.Gpu
@@ -21,6 +22,43 @@ namespace Ryujinx.Core.Gpu
             throw new NotImplementedException(Texture.Swizzle.ToString());
         }
 
+        public static int GetTextureSize(GalTexture Texture)
+        {
+            switch (Texture.Format)
+            {
+                case GalTextureFormat.R32G32B32A32: return Texture.Width * Texture.Height * 16;
+                case GalTextureFormat.R16G16B16A16: return Texture.Width * Texture.Height * 8;
+                case GalTextureFormat.A8B8G8R8:     return Texture.Width * Texture.Height * 4;
+                case GalTextureFormat.R32:          return Texture.Width * Texture.Height * 4;
+                case GalTextureFormat.A1B5G5R5:     return Texture.Width * Texture.Height * 2;
+                case GalTextureFormat.B5G6R5:       return Texture.Width * Texture.Height * 2;
+                case GalTextureFormat.G8R8:         return Texture.Width * Texture.Height * 2;
+                case GalTextureFormat.R8:           return Texture.Width * Texture.Height;
+
+                case GalTextureFormat.BC1:
+                case GalTextureFormat.BC4:
+                {
+                    int W = (Texture.Width  + 3) / 4;
+                    int H = (Texture.Height + 3) / 4;
+
+                    return W * H * 8;
+                }
+
+                case GalTextureFormat.BC2:
+                case GalTextureFormat.BC3:
+                case GalTextureFormat.BC5:
+                case GalTextureFormat.Astc2D4x4:
+                {
+                    int W = (Texture.Width  + 3) / 4;
+                    int H = (Texture.Height + 3) / 4;
+
+                    return W * H * 16;
+                }
+            }
+
+            throw new NotImplementedException(Texture.Format.ToString());
+        }
+
         public static (AMemory Memory, long Position) GetMemoryAndPosition(
             IAMemory Memory,
             long     Position)

Ryujinx.Core/Gpu/TextureReader.cs

@@ -4,7 +4,7 @@ using System;
 
 namespace Ryujinx.Core.Gpu
 {
-    public static class TextureReader
+    static class TextureReader
     {
         public static byte[] Read(IAMemory Memory, Texture Texture)
         {

Ryujinx.Core/Gpu/TextureSwizzle.cs

@@ -1,6 +1,6 @@
 namespace Ryujinx.Core.Gpu
 {
-    public enum TextureSwizzle
+    enum TextureSwizzle
     {
         _1dBuffer           = 0,
         PitchColorKey       = 1,

Ryujinx.Core/Gpu/TextureWriter.cs

@@ -4,7 +4,7 @@ using System;
 
 namespace Ryujinx.Core.Gpu
 {
-    public static class TextureWriter
+    static class TextureWriter
     {
         public static void Write(IAMemory Memory, Texture Texture, byte[] Data)
         {

Ryujinx.Core/OsHle/Kernel/SvcThread.cs

@@ -1,6 +1,7 @@
 using ChocolArm64.State;
 using Ryujinx.Core.Logging;
 using Ryujinx.Core.OsHle.Handles;
+using System;
 using System.Threading;
 
 using static Ryujinx.Core.OsHle.ErrorCode;