diff --git a/src/Ryujinx.Common/Memory/BytesReadOnlySequenceSegment.cs b/src/Ryujinx.Common/Memory/BytesReadOnlySequenceSegment.cs deleted file mode 100644 index 9c4cd1ff36..0000000000 --- a/src/Ryujinx.Common/Memory/BytesReadOnlySequenceSegment.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -using System.Buffers; - -namespace Ryujinx.Common.Memory -{ - /// - /// A concrete implementation of , - /// with methods to help build a full sequence. - /// - public sealed class BytesReadOnlySequenceSegment : ReadOnlySequenceSegment - { - public BytesReadOnlySequenceSegment(Memory memory) => Memory = memory; - - public BytesReadOnlySequenceSegment Append(Memory memory) - { - var nextSegment = new BytesReadOnlySequenceSegment(memory) - { - RunningIndex = RunningIndex + Memory.Length - }; - - Next = nextSegment; - - return nextSegment; - } - } -} diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs index d930dbc3bf..35812b355a 100644 --- a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs +++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs @@ -1,5 +1,4 @@ using ARMeilleure.Memory; -using Ryujinx.Common.Memory; using Ryujinx.Memory; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; @@ -8,7 +7,6 @@ using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Runtime.Versioning; namespace Ryujinx.Cpu.AppleHv @@ -17,7 +15,7 @@ namespace Ryujinx.Cpu.AppleHv /// Represents a CPU memory manager which maps guest virtual memory directly onto the Hypervisor page table. /// [SupportedOSPlatform("macos")] - public class HvMemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public class HvMemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private readonly InvalidAccessHandler _invalidAccessHandler; @@ -98,12 +96,6 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -128,20 +120,11 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - /// - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -154,7 +137,6 @@ namespace Ryujinx.Cpu.AppleHv } } - /// public override void Read(ulong va, Span data) { try @@ -170,101 +152,11 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - SignalMemoryTracking(va, (ulong)data.Length, true); - - WriteImpl(va, data); - } - - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - WriteImpl(va, data); - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return false; - } - - SignalMemoryTracking(va, (ulong)data.Length, false); - - if (IsContiguousAndMapped(va, data.Length)) - { - var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length); - - bool changed = !data.SequenceEqual(target); - - if (changed) - { - data.CopyTo(target); - } - - return changed; - } - else - { - WriteImpl(va, data); - - return true; - } - } - - private void WriteImpl(ulong va, ReadOnlySpan data) + public override void Write(ulong va, ReadOnlySpan data) { try { - AssertValidAddressAndSize(va, (ulong)data.Length); - - if (IsContiguousAndMapped(va, data.Length)) - { - data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length)); - } - else - { - int offset = 0, size; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressChecked(va); - - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - data[..size].CopyTo(_backingMemory.GetSpan(pa, size)); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); - - size = Math.Min(data.Length - offset, PageSize); - - data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size)); - } - } + base.Write(va, data); } catch (InvalidMemoryRegionException) { @@ -275,131 +167,38 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + public override void WriteUntracked(ulong va, ReadOnlySpan data) { - if (size == 0) + try { + base.WriteUntracked(va, data); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + { + try + { + return base.GetReadOnlySequence(va, size, tracked); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + return ReadOnlySequence.Empty; } - - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, false); - } - - if (IsContiguousAndMapped(va, size)) - { - return new ReadOnlySequence(_backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); - } - else - { - BytesReadOnlySequenceSegment first = null, last = null; - - try - { - AssertValidAddressAndSize(va, (ulong)size); - - int offset = 0, segmentSize; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressChecked(va); - - segmentSize = Math.Min(size, PageSize - (int)(va & PageMask)); - - var memory = _backingMemory.GetMemory(pa, segmentSize); - - first = last = new BytesReadOnlySequenceSegment(memory); - - offset += segmentSize; - } - - for (; offset < size; offset += segmentSize) - { - ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); - - segmentSize = Math.Min(size - offset, PageSize); - - var memory = _backingMemory.GetMemory(pa, segmentSize); - - if (first == null) - { - first = last = new BytesReadOnlySequenceSegment(memory); - } - else - { - last = last.Append(memory); - } - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - - return new ReadOnlySequence(first, 0, last, (int)(size - last.RunningIndex)); - } } - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return ReadOnlySpan.Empty; - } - - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, false); - } - - if (IsContiguousAndMapped(va, size)) - { - return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size); - } - else - { - Span data = new byte[size]; - - base.Read(va, data); - - return data; - } - } - - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return new WritableRegion(null, va, Memory.Empty); - } - - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, true); - } - - if (IsContiguousAndMapped(va, size)) - { - return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); - } - else - { - IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size); - - base.Read(va, memoryOwner.Memory.Span); - - return new WritableRegion(this, va, memoryOwner); - } - } - - /// public ref T GetRef(ulong va) where T : unmanaged { if (!IsContiguous(va, Unsafe.SizeOf())) @@ -412,9 +211,8 @@ namespace Ryujinx.Cpu.AppleHv return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va)); } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { return ValidateAddress(va) && _pages.IsMapped(va); } @@ -427,39 +225,6 @@ namespace Ryujinx.Cpu.AppleHv return _pages.IsRangeMapped(va, size); } - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguous(ulong va, int size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) - { - return false; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return false; - } - - if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize)) - { - return false; - } - - va += PageSize; - } - - return true; - } - /// public IEnumerable GetHostRegions(ulong va, ulong size) { @@ -536,11 +301,10 @@ namespace Ryujinx.Cpu.AppleHv return regions; } - /// /// /// This function also validates that the given range is both valid and mapped, and will throw if it is not. /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -553,24 +317,6 @@ namespace Ryujinx.Cpu.AppleHv _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, ulong size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - - /// public void Reprotect(ulong va, ulong size, MemoryPermission protection) { // TODO @@ -607,7 +353,7 @@ namespace Ryujinx.Cpu.AppleHv return Tracking.BeginSmartGranularTracking(address, size, granularity, id); } - private ulong GetPhysicalAddressChecked(ulong va) + private nuint GetPhysicalAddressChecked(ulong va) { if (!IsMapped(va)) { @@ -617,9 +363,9 @@ namespace Ryujinx.Cpu.AppleHv return GetPhysicalAddressInternal(va); } - private ulong GetPhysicalAddressInternal(ulong va) + private nuint GetPhysicalAddressInternal(ulong va) { - return _pageTable.Read(va) + (va & PageMask); + return (nuint)(_pageTable.Read(va) + (va & PageMask)); } /// @@ -630,10 +376,17 @@ namespace Ryujinx.Cpu.AppleHv _addressSpace.Dispose(); } - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _backingMemory.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) + protected override nuint TranslateVirtualAddressChecked(ulong va) => GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => GetPhysicalAddressInternal(va); + } } diff --git a/src/Ryujinx.Cpu/Jit/MemoryManager.cs b/src/Ryujinx.Cpu/Jit/MemoryManager.cs index ffed754bf3..dfa5b93539 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManager.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManager.cs @@ -1,5 +1,4 @@ using ARMeilleure.Memory; -using Ryujinx.Common.Memory; using Ryujinx.Memory; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; @@ -16,7 +15,7 @@ namespace Ryujinx.Cpu.Jit /// /// Represents a CPU memory manager. /// - public sealed class MemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public sealed class MemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private const int PteSize = 8; @@ -99,12 +98,6 @@ namespace Ryujinx.Cpu.Jit Tracking.Map(oVa, size); } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -130,20 +123,11 @@ namespace Ryujinx.Cpu.Jit } } - /// - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - /// - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -192,117 +176,11 @@ namespace Ryujinx.Cpu.Jit } } - /// - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - SignalMemoryTracking(va, (ulong)data.Length, true); - - WriteImpl(va, data); - } - - /// - public void WriteGuest(ulong va, T value) where T : unmanaged - { - Span data = MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)); - - SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true); - - WriteImpl(va, data); - } - - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - WriteImpl(va, data); - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return false; - } - - SignalMemoryTracking(va, (ulong)data.Length, false); - - if (IsContiguousAndMapped(va, data.Length)) - { - var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length); - - bool changed = !data.SequenceEqual(target); - - if (changed) - { - data.CopyTo(target); - } - - return changed; - } - else - { - WriteImpl(va, data); - - return true; - } - } - - /// - /// Writes data to CPU mapped memory. - /// - /// Virtual address to write the data into - /// Data to be written - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void WriteImpl(ulong va, ReadOnlySpan data) + public override void Write(ulong va, ReadOnlySpan data) { try { - AssertValidAddressAndSize(va, (ulong)data.Length); - - if (IsContiguousAndMapped(va, data.Length)) - { - data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length)); - } - else - { - int offset = 0, size; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressInternal(va); - - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - data[..size].CopyTo(_backingMemory.GetSpan(pa, size)); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - ulong pa = GetPhysicalAddressInternal(va + (ulong)offset); - - size = Math.Min(data.Length - offset, PageSize); - - data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size)); - } - } + base.Write(va, data); } catch (InvalidMemoryRegionException) { @@ -314,130 +192,47 @@ namespace Ryujinx.Cpu.Jit } /// - public ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + public void WriteGuest(ulong va, T value) where T : unmanaged { - if (size == 0) + Span data = MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)); + + SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true); + + Write(va, data); + } + + public override void WriteUntracked(ulong va, ReadOnlySpan data) + { + try { + base.WriteUntracked(va, data); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + { + try + { + return base.GetReadOnlySequence(va, size, tracked); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + return ReadOnlySequence.Empty; } - - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, false); - } - - if (IsContiguousAndMapped(va, size)) - { - return new ReadOnlySequence(_backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); - } - else - { - BytesReadOnlySequenceSegment first = null, last = null; - - try - { - AssertValidAddressAndSize(va, (ulong)size); - - int offset = 0, segmentSize; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressInternal(va); - - segmentSize = Math.Min(size, PageSize - (int)(va & PageMask)); - - var memory = _backingMemory.GetMemory(pa, segmentSize); - - first = last = new BytesReadOnlySequenceSegment(memory); - - offset += segmentSize; - } - - for (; offset < size; offset += segmentSize) - { - ulong pa = GetPhysicalAddressInternal(va + (ulong)offset); - - segmentSize = Math.Min(size - offset, PageSize); - - var memory = _backingMemory.GetMemory(pa, segmentSize); - - if (first == null) - { - first = last = new BytesReadOnlySequenceSegment(memory); - } - else - { - last = last.Append(memory); - } - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - - return new ReadOnlySequence(first, 0, last, (int)(size - last.RunningIndex)); - } } - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return ReadOnlySpan.Empty; - } - - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, false); - } - - if (IsContiguousAndMapped(va, size)) - { - return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size); - } - else - { - Span data = new byte[size]; - - base.Read(va, data); - - return data; - } - } - - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return new WritableRegion(null, va, Memory.Empty); - } - - if (IsContiguousAndMapped(va, size)) - { - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, true); - } - - return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); - } - else - { - IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size); - - GetSpan(va, size).CopyTo(memoryOwner.Memory.Span); - - return new WritableRegion(this, va, memoryOwner, tracked); - } - } - - /// public ref T GetRef(ulong va) where T : unmanaged { if (!IsContiguous(va, Unsafe.SizeOf())) @@ -450,56 +245,6 @@ namespace Ryujinx.Cpu.Jit return ref _backingMemory.GetRef(GetPhysicalAddressInternal(va)); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, uint size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguous(ulong va, int size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) - { - return false; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return false; - } - - if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize)) - { - return false; - } - - va += PageSize; - } - - return true; - } - /// public IEnumerable GetHostRegions(ulong va, ulong size) { @@ -604,9 +349,8 @@ namespace Ryujinx.Cpu.Jit return true; } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { if (!ValidateAddress(va)) { @@ -616,9 +360,9 @@ namespace Ryujinx.Cpu.Jit return _pageTable.Read((va / PageSize) * PteSize) != 0; } - private ulong GetPhysicalAddressInternal(ulong va) + private nuint GetPhysicalAddressInternal(ulong va) { - return PteToPa(_pageTable.Read((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask); + return (nuint)(PteToPa(_pageTable.Read((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask)); } /// @@ -715,9 +459,7 @@ namespace Ryujinx.Cpu.Jit { ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize); - long pte; - - pte = Volatile.Read(ref pageRef); + long pte = Volatile.Read(ref pageRef); if ((pte & tag) != 0) { @@ -735,7 +477,7 @@ namespace Ryujinx.Cpu.Jit } /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { SignalMemoryTrackingImpl(va, size, write, false, precise, exemptId); } @@ -755,10 +497,16 @@ namespace Ryujinx.Cpu.Jit /// protected override void Destroy() => _pageTable.Dispose(); - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _backingMemory.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) + protected override nuint TranslateVirtualAddressChecked(ulong va) + => GetPhysicalAddressInternal(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) => GetPhysicalAddressInternal(va); } } diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs index 3b356e2f12..c60ab6b246 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs @@ -13,7 +13,7 @@ namespace Ryujinx.Cpu.Jit /// /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. /// - public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private readonly InvalidAccessHandler _invalidAccessHandler; private readonly bool _unsafeMode; @@ -97,12 +97,6 @@ namespace Ryujinx.Cpu.Jit Tracking.Map(va, size); } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -139,8 +133,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public T Read(ulong va) where T : unmanaged + public override T Read(ulong va) { try { @@ -159,14 +152,11 @@ namespace Ryujinx.Cpu.Jit } } - /// - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -179,7 +169,6 @@ namespace Ryujinx.Cpu.Jit } } - /// public override void Read(ulong va, Span data) { try @@ -197,9 +186,7 @@ namespace Ryujinx.Cpu.Jit } } - - /// - public void Write(ulong va, T value) where T : unmanaged + public override void Write(ulong va, T value) { try { @@ -216,8 +203,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public void Write(ulong va, ReadOnlySpan data) + public override void Write(ulong va, ReadOnlySpan data) { try { @@ -234,8 +220,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) + public override void WriteUntracked(ulong va, ReadOnlySpan data) { try { @@ -252,8 +237,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) { try { @@ -280,8 +264,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) { if (tracked) { @@ -295,8 +278,7 @@ namespace Ryujinx.Cpu.Jit return new ReadOnlySequence(_addressSpace.Mirror.GetMemory(va, size)); } - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + public override ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) { if (tracked) { @@ -310,8 +292,7 @@ namespace Ryujinx.Cpu.Jit return _addressSpace.Mirror.GetSpan(va, size); } - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) { if (tracked) { @@ -325,7 +306,6 @@ namespace Ryujinx.Cpu.Jit return _addressSpace.Mirror.GetWritableRegion(va, size); } - /// public ref T GetRef(ulong va) where T : unmanaged { SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); @@ -333,9 +313,8 @@ namespace Ryujinx.Cpu.Jit return ref _addressSpace.Mirror.GetRef(va); } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { return ValidateAddress(va) && _pages.IsMapped(va); } @@ -406,11 +385,10 @@ namespace Ryujinx.Cpu.Jit return _pageTable.Read(va) + (va & PageMask); } - /// /// /// This function also validates that the given range is both valid and mapped, and will throw if it is not. /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -423,23 +401,6 @@ namespace Ryujinx.Cpu.Jit _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, ulong size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - /// public void Reprotect(ulong va, ulong size, MemoryPermission protection) { @@ -486,10 +447,16 @@ namespace Ryujinx.Cpu.Jit _memoryEh.Dispose(); } - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _addressSpace.Mirror.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _addressSpace.Mirror.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) - => va; + protected override nuint TranslateVirtualAddressChecked(ulong va) + => (nuint)GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => (nuint)GetPhysicalAddressInternal(va); } } diff --git a/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs b/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs index c2d8cfb1a0..3c7b338055 100644 --- a/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs +++ b/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs @@ -1,13 +1,10 @@ using Ryujinx.Memory; using System.Diagnostics; -using System.Numerics; using System.Threading; namespace Ryujinx.Cpu { - public abstract class VirtualMemoryManagerRefCountedBase : VirtualMemoryManagerBase, IRefCounted - where TVirtual : IBinaryInteger - where TPhysical : IBinaryInteger + public abstract class VirtualMemoryManagerRefCountedBase : VirtualMemoryManagerBase, IRefCounted { private int _referenceCount; diff --git a/src/Ryujinx.Memory/AddressSpaceManager.cs b/src/Ryujinx.Memory/AddressSpaceManager.cs index 4f79fdbb57..f089c85736 100644 --- a/src/Ryujinx.Memory/AddressSpaceManager.cs +++ b/src/Ryujinx.Memory/AddressSpaceManager.cs @@ -1,11 +1,8 @@ -using Ryujinx.Common.Memory; using Ryujinx.Memory.Range; using System; -using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace Ryujinx.Memory { @@ -13,7 +10,7 @@ namespace Ryujinx.Memory /// Represents a address space manager. /// Supports virtual memory region mapping, address translation and read/write access to mapped regions. /// - public sealed class AddressSpaceManager : VirtualMemoryManagerBase, IVirtualMemoryManager, IWritableBlock + public sealed class AddressSpaceManager : VirtualMemoryManagerBase, IVirtualMemoryManager { /// public bool Supports4KBPages => true; @@ -65,8 +62,7 @@ namespace Ryujinx.Memory } } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) + public override void MapForeign(ulong va, nuint hostPointer, ulong size) { AssertValidAddressAndSize(va, size); @@ -94,157 +90,6 @@ namespace Ryujinx.Memory } } - /// - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - /// - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - AssertValidAddressAndSize(va, (ulong)data.Length); - - if (IsContiguousAndMapped(va, data.Length)) - { - data.CopyTo(GetHostSpanContiguous(va, data.Length)); - } - else - { - int offset = 0, size; - - if ((va & PageMask) != 0) - { - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - data[..size].CopyTo(GetHostSpanContiguous(va, size)); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - size = Math.Min(data.Length - offset, PageSize); - - data.Slice(offset, size).CopyTo(GetHostSpanContiguous(va + (ulong)offset, size)); - } - } - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - Write(va, data); - - return true; - } - - /// - public ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return ReadOnlySequence.Empty; - } - - if (IsContiguousAndMapped(va, size)) - { - return new ReadOnlySequence(GetHostMemoryContiguous(va, size)); - } - else - { - AssertValidAddressAndSize(va, (ulong)size); - - int offset = 0, segmentSize; - - BytesReadOnlySequenceSegment first = null, last = null; - - if ((va & PageMask) != 0) - { - segmentSize = Math.Min(size, PageSize - (int)(va & PageMask)); - - var memory = GetHostMemoryContiguous(va, segmentSize); - - first = last = new BytesReadOnlySequenceSegment(memory); - - offset += segmentSize; - } - - for (; offset < size; offset += segmentSize) - { - segmentSize = Math.Min(size - offset, PageSize); - - var memory = GetHostMemoryContiguous(va + (ulong)offset, segmentSize); - - if (first == null) - { - first = last = new BytesReadOnlySequenceSegment(memory); - } - else - { - last = last.Append(memory); - } - } - - return new ReadOnlySequence(first, 0, last, (int)(size - last.RunningIndex)); - } - } - - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return ReadOnlySpan.Empty; - } - - if (IsContiguousAndMapped(va, size)) - { - return GetHostSpanContiguous(va, size); - } - else - { - Span data = new byte[size]; - - Read(va, data); - - return data; - } - } - - /// - public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return new WritableRegion(null, va, Memory.Empty); - } - - if (IsContiguousAndMapped(va, size)) - { - return new WritableRegion(null, va, GetHostMemoryContiguous(va, size)); - } - else - { - IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size); - - GetSpan(va, size).CopyTo(memoryOwner.Memory.Span); - - return new WritableRegion(this, va, memoryOwner); - } - } - /// public unsafe ref T GetRef(ulong va) where T : unmanaged { @@ -256,50 +101,6 @@ namespace Ryujinx.Memory return ref *(T*)GetHostAddress(va); } - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, uint size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguous(ulong va, int size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) - { - return false; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return false; - } - - if (GetHostAddress(va) + PageSize != GetHostAddress(va + PageSize)) - { - return false; - } - - va += PageSize; - } - - return true; - } - /// public IEnumerable GetHostRegions(ulong va, ulong size) { @@ -357,7 +158,7 @@ namespace Ryujinx.Memory return null; } - int pages = GetPagesCount(va, (uint)size, out va); + int pages = GetPagesCount(va, size, out va); var regions = new List(); @@ -389,9 +190,8 @@ namespace Ryujinx.Memory return regions; } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { if (!ValidateAddress(va)) { @@ -404,7 +204,7 @@ namespace Ryujinx.Memory /// public bool IsRangeMapped(ulong va, ulong size) { - if (size == 0UL) + if (size == 0) { return true; } @@ -429,16 +229,6 @@ namespace Ryujinx.Memory return true; } - private unsafe Memory GetHostMemoryContiguous(ulong va, int size) - { - return new NativeMemoryManager((byte*)GetHostAddress(va), size).Memory; - } - - private unsafe Span GetHostSpanContiguous(ulong va, int size) - { - return new Span((void*)GetHostAddress(va), size); - } - private nuint GetHostAddress(ulong va) { return _pageTable.Read(va) + (nuint)(va & PageMask); @@ -455,16 +245,16 @@ namespace Ryujinx.Memory throw new NotImplementedException(); } - /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) - { - // Only the ARM Memory Manager has tracking for now. - } + protected unsafe override Memory GetPhysicalAddressMemory(nuint pa, int size) + => new NativeMemoryManager((byte*)pa, size).Memory; protected override unsafe Span GetPhysicalAddressSpan(nuint pa, int size) - => new((void*)pa, size); + => new Span((void*)pa, size); - protected override nuint TranslateVirtualAddressForRead(ulong va) + protected override nuint TranslateVirtualAddressChecked(ulong va) + => GetHostAddress(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) => GetHostAddress(va); } } diff --git a/src/Ryujinx.Memory/BytesReadOnlySequenceSegment.cs b/src/Ryujinx.Memory/BytesReadOnlySequenceSegment.cs new file mode 100644 index 0000000000..5fe8d936c3 --- /dev/null +++ b/src/Ryujinx.Memory/BytesReadOnlySequenceSegment.cs @@ -0,0 +1,60 @@ +using System; +using System.Buffers; +using System.Runtime.InteropServices; + +namespace Ryujinx.Memory +{ + /// + /// A concrete implementation of , + /// with methods to help build a full sequence. + /// + public sealed class BytesReadOnlySequenceSegment : ReadOnlySequenceSegment + { + public BytesReadOnlySequenceSegment(Memory memory) => Memory = memory; + + public BytesReadOnlySequenceSegment Append(Memory memory) + { + var nextSegment = new BytesReadOnlySequenceSegment(memory) + { + RunningIndex = RunningIndex + Memory.Length + }; + + Next = nextSegment; + + return nextSegment; + } + + /// + /// Attempts to determine if the current and are contiguous. + /// Only works if both were created by a . + /// + /// The segment to check if continuous with the current one + /// The starting address of the contiguous segment + /// The size of the contiguous segment + /// True if the segments are contiguous, otherwise false + public unsafe bool IsContiguousWith(Memory other, out nuint contiguousStart, out int contiguousSize) + { + if (MemoryMarshal.TryGetMemoryManager>(Memory, out var thisMemoryManager) && + MemoryMarshal.TryGetMemoryManager>(other, out var otherMemoryManager) && + thisMemoryManager.Pointer + thisMemoryManager.Length == otherMemoryManager.Pointer) + { + contiguousStart = (nuint)thisMemoryManager.Pointer; + contiguousSize = thisMemoryManager.Length + otherMemoryManager.Length; + return true; + } + else + { + contiguousStart = 0; + contiguousSize = 0; + return false; + } + } + + /// + /// Replaces the current value with the one provided. + /// + /// The new segment to hold in this + public void Replace(Memory memory) + => Memory = memory; + } +} diff --git a/src/Ryujinx.Memory/NativeMemoryManager.cs b/src/Ryujinx.Memory/NativeMemoryManager.cs index fe718bda81..9ca6329382 100644 --- a/src/Ryujinx.Memory/NativeMemoryManager.cs +++ b/src/Ryujinx.Memory/NativeMemoryManager.cs @@ -14,6 +14,10 @@ namespace Ryujinx.Memory _length = length; } + public unsafe T* Pointer => _pointer; + + public int Length => _length; + public override Span GetSpan() { return new Span((void*)_pointer, _length); diff --git a/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs b/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs index cbec88cc56..a99c577292 100644 --- a/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs +++ b/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs @@ -1,34 +1,171 @@ +using Ryujinx.Common.Memory; using System; -using System.Numerics; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Ryujinx.Memory { - public abstract class VirtualMemoryManagerBase - where TVirtual : IBinaryInteger - where TPhysical : IBinaryInteger + public abstract class VirtualMemoryManagerBase : IWritableBlock { public const int PageBits = 12; public const int PageSize = 1 << PageBits; public const int PageMask = PageSize - 1; - protected abstract TVirtual AddressSpaceSize { get; } + protected abstract ulong AddressSpaceSize { get; } - public virtual void Read(TVirtual va, Span data) + public virtual ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySequence.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return new ReadOnlySequence(GetPhysicalAddressMemory(pa, size)); + } + else + { + AssertValidAddressAndSize(va, size); + + int offset = 0, segmentSize; + + BytesReadOnlySequenceSegment first = null, last = null; + + if ((va & PageMask) != 0) + { + nuint pa = TranslateVirtualAddressChecked(va); + + segmentSize = Math.Min(size, PageSize - (int)(va & PageMask)); + + Memory memory = GetPhysicalAddressMemory(pa, segmentSize); + + first = last = new BytesReadOnlySequenceSegment(memory); + + offset += segmentSize; + } + + for (; offset < size; offset += segmentSize) + { + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); + + segmentSize = Math.Min(size - offset, PageSize); + + Memory memory = GetPhysicalAddressMemory(pa, segmentSize); + + if (first is null) + { + first = last = new BytesReadOnlySequenceSegment(memory); + } + else + { + if (last.IsContiguousWith(memory, out nuint contiguousStart, out int contiguousSize)) + { + last.Replace(GetPhysicalAddressMemory(contiguousStart, contiguousSize)); + } + else + { + last = last.Append(memory); + } + } + } + + return new ReadOnlySequence(first, 0, last, (int)(size - last.RunningIndex)); + } + } + + public virtual ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySpan.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return GetPhysicalAddressSpan(pa, size); + } + else + { + Span data = new byte[size]; + + Read(va, data); + + return data; + } + } + + public virtual WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return new WritableRegion(null, va, Memory.Empty); + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, true); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return new WritableRegion(null, va, GetPhysicalAddressMemory(pa, size)); + } + else + { + IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size); + + Read(va, memoryOwner.Memory.Span); + + return new WritableRegion(this, va, memoryOwner); + } + } + + public abstract bool IsMapped(ulong va); + + public virtual void MapForeign(ulong va, nuint hostPointer, ulong size) + { + throw new NotSupportedException(); + } + + public virtual T Read(ulong va) where T : unmanaged + { + return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; + } + + public virtual void Read(ulong va, Span data) { if (data.Length == 0) { return; } - AssertValidAddressAndSize(va, TVirtual.CreateChecked(data.Length)); + AssertValidAddressAndSize(va, data.Length); int offset = 0, size; - if ((int.CreateTruncating(va) & PageMask) != 0) + if ((va & PageMask) != 0) { - TPhysical pa = TranslateVirtualAddressForRead(va); + nuint pa = TranslateVirtualAddressChecked(va); - size = Math.Min(data.Length, PageSize - ((int.CreateTruncating(va) & PageMask))); + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); GetPhysicalAddressSpan(pa, size).CopyTo(data[..size]); @@ -37,7 +174,7 @@ namespace Ryujinx.Memory for (; offset < data.Length; offset += size) { - TPhysical pa = TranslateVirtualAddressForRead(va + TVirtual.CreateChecked(offset)); + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); size = Math.Min(data.Length - offset, PageSize); @@ -45,13 +182,84 @@ namespace Ryujinx.Memory } } + public virtual T ReadTracked(ulong va) where T : unmanaged + { + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); + + return Read(va); + } + + public virtual void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + // No default implementation + } + + public virtual void Write(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + SignalMemoryTracking(va, (ulong)data.Length, true); + + WriteImpl(va, data); + } + + public virtual void Write(ulong va, T value) where T : unmanaged + { + Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); + } + + public virtual void WriteUntracked(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + WriteImpl(va, data); + } + + public virtual bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return false; + } + + if (IsContiguousAndMapped(va, data.Length)) + { + SignalMemoryTracking(va, (ulong)data.Length, false); + + nuint pa = TranslateVirtualAddressChecked(va); + + var target = GetPhysicalAddressSpan(pa, data.Length); + + bool changed = !data.SequenceEqual(target); + + if (changed) + { + data.CopyTo(target); + } + + return changed; + } + else + { + Write(va, data); + + return true; + } + } + /// /// Ensures the combination of virtual address and size is part of the addressable space. /// /// Virtual address of the range /// Size of the range in bytes /// Throw when the memory region specified outside the addressable space - protected void AssertValidAddressAndSize(TVirtual va, TVirtual size) + protected void AssertValidAddressAndSize(ulong va, ulong size) { if (!ValidateAddressAndSize(va, size)) { @@ -59,16 +267,84 @@ namespace Ryujinx.Memory } } - protected abstract Span GetPhysicalAddressSpan(TPhysical pa, int size); + /// + /// Ensures the combination of virtual address and size is part of the addressable space. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// Throw when the memory region specified outside the addressable space + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected void AssertValidAddressAndSize(ulong va, int size) + => AssertValidAddressAndSize(va, (ulong)size); - protected abstract TPhysical TranslateVirtualAddressForRead(TVirtual va); + /// + /// Computes the number of pages in a virtual address range. + /// + /// Virtual address of the range + /// Size of the range + /// The virtual address of the beginning of the first page + /// This function does not differentiate between allocated and unallocated pages. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected static int GetPagesCount(ulong va, ulong size, out ulong startVa) + { + // WARNING: Always check if ulong does not overflow during the operations. + startVa = va & ~(ulong)PageMask; + ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; + + return (int)(vaSpan / PageSize); + } + + protected abstract Memory GetPhysicalAddressMemory(nuint pa, int size); + + protected abstract Span GetPhysicalAddressSpan(nuint pa, int size); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool IsContiguous(ulong va, int size) => IsContiguous(va, (ulong)size); + + protected virtual bool IsContiguous(ulong va, ulong size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) + { + return false; + } + + int pages = GetPagesCount(va, size, out va); + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return false; + } + + if (TranslateVirtualAddressUnchecked(va) + PageSize != TranslateVirtualAddressUnchecked(va + PageSize)) + { + return false; + } + + va += PageSize; + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool IsContiguousAndMapped(ulong va, ulong size) => IsContiguous(va, size) && IsMapped(va); + + protected abstract nuint TranslateVirtualAddressChecked(ulong va); + + protected abstract nuint TranslateVirtualAddressUnchecked(ulong va); /// /// Checks if the virtual address is part of the addressable space. /// /// Virtual address /// True if the virtual address is part of the addressable space - protected bool ValidateAddress(TVirtual va) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool ValidateAddress(ulong va) { return va < AddressSpaceSize; } @@ -79,13 +355,53 @@ namespace Ryujinx.Memory /// Virtual address of the range /// Size of the range in bytes /// True if the combination of virtual address and size is part of the addressable space - protected bool ValidateAddressAndSize(TVirtual va, TVirtual size) + protected bool ValidateAddressAndSize(ulong va, ulong size) { - TVirtual endVa = va + size; + ulong endVa = va + size; return endVa >= va && endVa >= size && endVa <= AddressSpaceSize; } protected static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + + protected static void ThrowMemoryNotContiguous() + => throw new MemoryNotContiguousException(); + + private void WriteImpl(ulong va, ReadOnlySpan data) + { + AssertValidAddressAndSize(va, data.Length); + + if (IsContiguousAndMapped(va, data.Length)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + data.CopyTo(GetPhysicalAddressSpan(pa, data.Length)); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + nuint pa = TranslateVirtualAddressChecked(va); + + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); + + data[..size].CopyTo(GetPhysicalAddressSpan(pa, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); + + size = Math.Min(data.Length - offset, PageSize); + + data.Slice(offset, size).CopyTo(GetPhysicalAddressSpan(pa, size)); + } + } + } + } }