mirrors/Ryujinx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement faster address translation and write tracking on the MMU

Browse source
This commit is contained in:
gdkchan 2019-02-19 16:32:13 -03:00
parent 6335753e38
commit 0d75d1d25f
16 changed files with 688 additions and 464 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
ChocolArm64/Events/InvalidAccessEventArgs.cs
View file

@ -1,14 +0,0 @@
using System;
namespace ChocolArm64.Events
{
public class MemoryAccessEventArgs : EventArgs
{
public long Position { get; private set; }
public MemoryAccessEventArgs(long position)
{
Position = position;
}
}
}

13
ChocolArm64/Exceptions/VmmPageFaultException.cs
View file

@ -1,13 +0,0 @@
using System;
namespace ChocolArm64.Exceptions
{
public class VmmPageFaultException : Exception
{
private const string ExMsg = "Tried to access unmapped address 0x{0:x16}!";
public VmmPageFaultException() { }
public VmmPageFaultException(long position) : base(string.Format(ExMsg, position)) { }
}
}

11
ChocolArm64/Instructions/InstEmitMemory.cs
View file

@ -31,8 +31,6 @@ namespace ChocolArm64.Instructions
{
OpCodeMem64 op = (OpCodeMem64)context.CurrOp;
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
EmitLoadAddress(context);
if (signed && op.Extend64)
@ -69,7 +67,6 @@ namespace ChocolArm64.Instructions
return;
}
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdc_I8(op.Imm);
if (op.Signed)
@ -116,13 +113,10 @@ namespace ChocolArm64.Instructions
}
}
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
EmitLoadAddress(context);
EmitReadAndStore(op.Rt);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I8(1 << op.Size);
@ -137,8 +131,6 @@ namespace ChocolArm64.Instructions
{
OpCodeMem64 op = (OpCodeMem64)context.CurrOp;
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
EmitLoadAddress(context);
if (op is IOpCodeSimd64)
@ -159,8 +151,6 @@ namespace ChocolArm64.Instructions
{
OpCodeMemPair64 op = (OpCodeMemPair64)context.CurrOp;
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
EmitLoadAddress(context);
if (op is IOpCodeSimd64)
@ -174,7 +164,6 @@ namespace ChocolArm64.Instructions
EmitWriteCall(context, op.Size);
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I8(1 << op.Size);

6
ChocolArm64/Instructions/InstEmitMemory32.cs
View file

@ -64,9 +64,7 @@ namespace ChocolArm64.Instructions
{
if ((mask & 1) != 0)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I4(offset);
context.Emit(OpCodes.Add);
@ -129,9 +127,7 @@ namespace ChocolArm64.Instructions
{
if ((mask & 1) != 0)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
context.EmitLdc_I4(offset);
context.Emit(OpCodes.Add);
@ -198,8 +194,6 @@ namespace ChocolArm64.Instructions
context.EmitSttmp();
}
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
if (op.Index)
{
context.EmitLdtmp();

20
ChocolArm64/Instructions/InstEmitMemoryEx.cs
View file

@ -72,6 +72,8 @@ namespace ChocolArm64.Instructions
void WriteExclusiveValue(string propName)
{
context.Emit(OpCodes.Dup);
if (op.Size < 3)
{
context.Emit(OpCodes.Conv_U8);
@ -82,13 +84,6 @@ namespace ChocolArm64.Instructions
context.EmitLdtmp2();
context.EmitCallPrivatePropSet(typeof(CpuThreadState), propName);
context.EmitLdtmp2();
if (op.Size < 3)
{
context.Emit(OpCodes.Conv_U4);
}
}
if (pair)
@ -99,7 +94,6 @@ namespace ChocolArm64.Instructions
//method to read 128-bits atomically.
if (op.Size == 2)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, 3);
@ -170,7 +164,6 @@ namespace ChocolArm64.Instructions
else
{
//8, 16, 32 or 64-bits (non-pairwise) load.
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdtmp();
EmitReadZxCall(context, op.Size);
@ -320,9 +313,8 @@ namespace ChocolArm64.Instructions
}
else
{
void EmitWrite(int rt, long offset)
void EmitWriteCall(int rt, long offset)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
if (offset != 0)
@ -334,14 +326,14 @@ namespace ChocolArm64.Instructions
context.EmitLdintzr(rt);
EmitWriteCall(context, op.Size);
InstEmitMemoryHelper.EmitWriteCall(context, op.Size);
}
EmitWrite(op.Rt, 0);
EmitWriteCall(op.Rt, 0);
if (pair)
{
EmitWrite(op.Rt2, 1 << op.Size);
EmitWriteCall(op.Rt2, 1 << op.Size);
}
}
}

347
ChocolArm64/Instructions/InstEmitMemoryHelper.cs
View file

@ -1,13 +1,20 @@
using ChocolArm64.Decoders;
using ChocolArm64.Memory;
using ChocolArm64.State;
using ChocolArm64.Translation;
using System;
using System.Reflection.Emit;
using System.Runtime.Intrinsics.X86;
namespace ChocolArm64.Instructions
{
static class InstEmitMemoryHelper
{
private static int _tempIntAddress = ILEmitterCtx.GetIntTempIndex();
private static int _tempIntValue = ILEmitterCtx.GetIntTempIndex();
private static int _tempIntPtAddr = ILEmitterCtx.GetIntTempIndex();
private static int _tempVecValue = ILEmitterCtx.GetVecTempIndex();
private enum Extension
{
Zx,
@ -32,9 +39,10 @@ namespace ChocolArm64.Instructions
private static void EmitReadCall(ILEmitterCtx context, Extension ext, int size)
{
bool isSimd = GetIsSimd(context);
//Save the address into a temp.
context.EmitStint(_tempIntAddress);
string name = null;
bool isSimd = IsSimd(context);
if (size < 0 || size > (isSimd ? 4 : 3))
{
@ -43,28 +51,27 @@ namespace ChocolArm64.Instructions
if (isSimd)
{
switch (size)
if (size != 4)
{
case 0: name = nameof(MemoryManager.ReadVector8); break;
case 1: name = nameof(MemoryManager.ReadVector16); break;
case 2: name = nameof(MemoryManager.ReadVector32); break;
case 3: name = nameof(MemoryManager.ReadVector64); break;
case 4: name = nameof(MemoryManager.ReadVector128); break;
EmitReadVectorFallback(context, size);
}
else
{
EmitReadVector(context, size);
}
}
else
{
switch (size)
if (context.Tier == TranslationTier.Tier0)
{
case 0: name = nameof(MemoryManager.ReadByte); break;
case 1: name = nameof(MemoryManager.ReadUInt16); break;
case 2: name = nameof(MemoryManager.ReadUInt32); break;
case 3: name = nameof(MemoryManager.ReadUInt64); break;
EmitReadIntFallback(context, size);
}
else
{
EmitReadInt(context, size);
}
}
context.EmitCall(typeof(MemoryManager), name);
if (!isSimd)
{
if (ext == Extension.Sx32 ||
@ -89,50 +96,318 @@ namespace ChocolArm64.Instructions
public static void EmitWriteCall(ILEmitterCtx context, int size)
{
bool isSimd = GetIsSimd(context);
bool isSimd = IsSimd(context);
string name = null;
//Save the value into a temp.
if (isSimd)
{
context.EmitStvec(_tempVecValue);
}
else
{
context.EmitStint(_tempIntValue);
}
//Save the address into a temp.
context.EmitStint(_tempIntAddress);
if (size < 0 || size > (isSimd ? 4 : 3))
{
throw new ArgumentOutOfRangeException(nameof(size));
}
if (size < 3 && !isSimd)
{
context.Emit(OpCodes.Conv_I4);
}
if (isSimd)
{
switch (size)
if (size != 4)
{
case 0: name = nameof(MemoryManager.WriteVector8); break;
case 1: name = nameof(MemoryManager.WriteVector16); break;
case 2: name = nameof(MemoryManager.WriteVector32); break;
case 3: name = nameof(MemoryManager.WriteVector64); break;
case 4: name = nameof(MemoryManager.WriteVector128); break;
EmitWriteVectorFallback(context, size);
}
else
{
EmitWriteVector(context, size);
}
}
else
{
switch (size)
if (context.Tier == TranslationTier.Tier0)
{
case 0: name = nameof(MemoryManager.WriteByte); break;
case 1: name = nameof(MemoryManager.WriteUInt16); break;
case 2: name = nameof(MemoryManager.WriteUInt32); break;
case 3: name = nameof(MemoryManager.WriteUInt64); break;
EmitWriteIntFallback(context, size);
}
else
{
EmitWriteInt(context, size);
}
}
context.EmitCall(typeof(MemoryManager), name);
}
private static bool GetIsSimd(ILEmitterCtx context)
private static bool IsSimd(ILEmitterCtx context)
{
return context.CurrOp is IOpCodeSimd64 &&
!(context.CurrOp is OpCodeSimdMemMs64 ||
context.CurrOp is OpCodeSimdMemSs64);
}
private static void EmitReadInt(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitReadIntFallback(context, size);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
switch (size)
{
case 0: context.Emit(OpCodes.Ldind_U1); break;
case 1: context.Emit(OpCodes.Ldind_U2); break;
case 2: context.Emit(OpCodes.Ldind_U4); break;
case 3: context.Emit(OpCodes.Ldind_I8); break;
}
context.MarkLabel(lblEnd);
}
private static void EmitReadVector(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitReadVectorFallback(context, size);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
context.EmitCall(typeof(Sse), nameof(Sse.LoadVector128));
context.MarkLabel(lblEnd);
}
private static void EmitWriteInt(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitWriteIntFallback(context, size);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
context.EmitLdint(_tempIntValue);
if (size < 3)
{
context.Emit(OpCodes.Conv_U4);
}
switch (size)
{
case 0: context.Emit(OpCodes.Stind_I1); break;
case 1: context.Emit(OpCodes.Stind_I2); break;
case 2: context.Emit(OpCodes.Stind_I4); break;
case 3: context.Emit(OpCodes.Stind_I8); break;
}
context.MarkLabel(lblEnd);
}
private static void EmitWriteVector(ILEmitterCtx context, int size)
{
EmitAddressCheck(context, size);
ILLabel lblFastPath = new ILLabel();
ILLabel lblSlowPath = new ILLabel();
ILLabel lblEnd = new ILLabel();
context.Emit(OpCodes.Brfalse_S, lblFastPath);
context.MarkLabel(lblSlowPath);
EmitWriteVectorFallback(context, size);
context.Emit(OpCodes.Br_S, lblEnd);
context.MarkLabel(lblFastPath);
EmitPtPointerLoad(context, lblSlowPath);
context.EmitLdvec(_tempVecValue);
context.EmitCall(typeof(Sse), nameof(Sse.Store));
context.MarkLabel(lblEnd);
}
private static void EmitAddressCheck(ILEmitterCtx context, int size)
{
long addressCheckMask = ~(context.Memory.AddressSpaceSize - 1);
addressCheckMask |= (1u << size) - 1;
context.EmitLdint(_tempIntAddress);
context.EmitLdc_I(addressCheckMask);
context.Emit(OpCodes.And);
}
private static void EmitPtPointerLoad(ILEmitterCtx context, ILLabel lblFallbackPath)
{
context.EmitLdint(_tempIntAddress);
context.EmitLsr(MemoryManager.PageBits);
context.EmitLdc_I(IntPtr.Size);
context.Emit(OpCodes.Mul);
if (context.CurrOp.RegisterSize == RegisterSize.Int32)
{
context.Emit(OpCodes.Conv_U8);
}
context.EmitLdc_I8(context.Memory.PageTable.ToInt64());
context.Emit(OpCodes.Add);
context.Emit(OpCodes.Conv_I);
context.Emit(OpCodes.Ldind_I);
if (!context.Memory.HasWriteWatchSupport)
{
context.Emit(OpCodes.Conv_U8);
context.EmitStint(_tempIntPtAddr);
context.EmitLdint(_tempIntPtAddr);
context.EmitLdc_I8(MemoryManager.PteFlagsMask);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Brtrue, lblFallbackPath);
context.EmitLdint(_tempIntPtAddr);
context.Emit(OpCodes.Conv_I);
}
context.EmitLdint(_tempIntAddress);
context.EmitLdc_I(MemoryManager.PageMask);
context.Emit(OpCodes.And);
context.Emit(OpCodes.Conv_I);
context.Emit(OpCodes.Add);
}
private static void EmitReadIntFallback(ILEmitterCtx context, int size)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(_tempIntAddress);
string fallbackMethodName = null;
switch (size)
{
case 0: fallbackMethodName = nameof(MemoryManager.ReadByte); break;
case 1: fallbackMethodName = nameof(MemoryManager.ReadUInt16); break;
case 2: fallbackMethodName = nameof(MemoryManager.ReadUInt32); break;
case 3: fallbackMethodName = nameof(MemoryManager.ReadUInt64); break;
}
context.EmitCall(typeof(MemoryManager), fallbackMethodName);
}
private static void EmitReadVectorFallback(ILEmitterCtx context, int size)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(_tempIntAddress);
string fallbackMethodName = null;
switch (size)
{
case 0: fallbackMethodName = nameof(MemoryManager.ReadVector8); break;
case 1: fallbackMethodName = nameof(MemoryManager.ReadVector16); break;
case 2: fallbackMethodName = nameof(MemoryManager.ReadVector32); break;
case 3: fallbackMethodName = nameof(MemoryManager.ReadVector64); break;
case 4: fallbackMethodName = nameof(MemoryManager.ReadVector128); break;
}
context.EmitCall(typeof(MemoryManager), fallbackMethodName);
}
private static void EmitWriteIntFallback(ILEmitterCtx context, int size)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(_tempIntAddress);
context.EmitLdint(_tempIntValue);
if (size < 3)
{
context.Emit(OpCodes.Conv_U4);
}
string fallbackMethodName = null;
switch (size)
{
case 0: fallbackMethodName = nameof(MemoryManager.WriteByte); break;
case 1: fallbackMethodName = nameof(MemoryManager.WriteUInt16); break;
case 2: fallbackMethodName = nameof(MemoryManager.WriteUInt32); break;
case 3: fallbackMethodName = nameof(MemoryManager.WriteUInt64); break;
}
context.EmitCall(typeof(MemoryManager), fallbackMethodName);
}
private static void EmitWriteVectorFallback(ILEmitterCtx context, int size)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(_tempIntAddress);
context.EmitLdvec(_tempVecValue);
string fallbackMethodName = null;
switch (size)
{
case 0: fallbackMethodName = nameof(MemoryManager.WriteVector8); break;
case 1: fallbackMethodName = nameof(MemoryManager.WriteVector16); break;
case 2: fallbackMethodName = nameof(MemoryManager.WriteVector32); break;
case 3: fallbackMethodName = nameof(MemoryManager.WriteVector64); break;
case 4: fallbackMethodName = nameof(MemoryManager.WriteVector128); break;
}
context.EmitCall(typeof(MemoryManager), fallbackMethodName);
}
}
}

3
ChocolArm64/Instructions/InstEmitSimdMemory.cs
View file

@ -45,7 +45,6 @@ namespace ChocolArm64.Instructions
if (isLoad)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdc_I8(offset);
@ -62,7 +61,6 @@ namespace ChocolArm64.Instructions
}
else
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdc_I8(offset);
@ -90,7 +88,6 @@ namespace ChocolArm64.Instructions
void EmitMemAddress()
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdint(op.Rn);
context.EmitLdc_I8(offset);

1
ChocolArm64/Instructions/InstEmitSystem.cs
View file

@ -102,7 +102,6 @@ namespace ChocolArm64.Instructions
//DC ZVA
for (int offs = 0; offs < (4 << CpuThreadState.DczSizeLog2); offs += 8)
{
context.EmitLdarg(TranslatedSub.MemoryArgIdx);
context.EmitLdintzr(op.Rt);
context.EmitLdc_I(offs);

611
ChocolArm64/Memory/MemoryManager.cs
View file

@ -1,8 +1,5 @@
using ChocolArm64.Events;
using ChocolArm64.Exceptions;
using ChocolArm64.Instructions;
using System;
using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
@ -10,52 +7,318 @@ using System.Runtime.Intrinsics.X86;
using System.Threading;
using static ChocolArm64.Memory.CompareExchange128;
using static ChocolArm64.Memory.MemoryAlloc;
namespace ChocolArm64.Memory
{
public unsafe class MemoryManager : IMemory, IDisposable
{
private const int PtLvl0Bits = 13;
private const int PtLvl1Bits = 14;
public const int PageBits = 12;
public const int PageBits = 12;
public const int PageSize = 1 << PageBits;
public const int PageMask = PageSize - 1;
private const int PtLvl0Size = 1 << PtLvl0Bits;
private const int PtLvl1Size = 1 << PtLvl1Bits;
public const int PageSize = 1 << PageBits;
private const long PteFlagNotModified = 1;
private const int PtLvl0Mask = PtLvl0Size - 1;
private const int PtLvl1Mask = PtLvl1Size - 1;
public const int PageMask = PageSize - 1;
private const int PtLvl0Bit = PageBits + PtLvl1Bits;
private const int PtLvl1Bit = PageBits;
private ConcurrentDictionary<long, IntPtr> _observedPages;
internal const long PteFlagsMask = 7;
public IntPtr Ram { get; private set; }
private byte* _ramPtr;
private byte*** _pageTable;
private IntPtr _pageTable;
public event EventHandler<MemoryAccessEventArgs> InvalidAccess;
internal IntPtr PageTable => _pageTable;
public event EventHandler<MemoryAccessEventArgs> ObservedAccess;
public bool HasWriteWatchSupport => MemoryAlloc.HasWriteWatchSupport;
public MemoryManager(IntPtr ram)
public long AddressSpaceSize { get; }
public MemoryManager(IntPtr ram, long addressSpaceSize = 1L << 39)
{
_observedPages = new ConcurrentDictionary<long, IntPtr>();
Ram = ram;
_ramPtr = (byte*)ram;
_pageTable = (byte***)Marshal.AllocHGlobal(PtLvl0Size * IntPtr.Size);
AddressSpaceSize = addressSpaceSize;
for (int l0 = 0; l0 < PtLvl0Size; l0++)
long pageTableSize = (addressSpaceSize / PageSize) * IntPtr.Size;
_pageTable = Allocate((ulong)pageTableSize);
}
public void Map(long va, long pa, long size)
{
SetPtEntries(va, _ramPtr + pa, size);
}
public void Unmap(long position, long size)
{
SetPtEntries(position, null, size);
}
public bool IsMapped(long position)
{
return Translate(position) != IntPtr.Zero;
}
public long GetPhysicalAddress(long virtualAddress)
{
byte* ptr = (byte*)Translate(virtualAddress);
return (long)(ptr - _ramPtr);
}
private IntPtr Translate(long position)
{
if (!IsValidPosition(position))
{
_pageTable[l0] = null;
return IntPtr.Zero;
}
byte* ptr = ((byte**)_pageTable)[position >> PageBits];
IntPtr* pt = (IntPtr*)_pageTable;
ulong ptrUlong = (ulong)ptr;
long ptrLong = (long)ptr;
if ((ptrUlong & PteFlagsMask) != 0)
{
ptrUlong &= ~(ulong)PteFlagsMask;
ptr = (byte*)ptrUlong;
}
return new IntPtr(ptr + (position & PageMask));
}
private IntPtr TranslateWrite(long position)
{
if (!IsValidPosition(position))
{
return IntPtr.Zero;
}
byte* ptr = ((byte**)_pageTable)[position >> PageBits];
ulong ptrUlong = (ulong)ptr;
if ((ptrUlong & PteFlagsMask) != 0)
{
if ((ptrUlong & PteFlagNotModified) != 0)
{
ClearPtEntryFlag(position, PteFlagNotModified);
}
ptrUlong &= ~(ulong)PteFlagsMask;
ptr = (byte*)ptrUlong;
}
return new IntPtr(ptr + (position & PageMask));
}
private void SetPtEntries(long va, byte* ptr, long size)
{
long endPosition = (va + size + PageMask) & ~PageMask;
while ((ulong)va < (ulong)endPosition)
{
SetPtEntry(va, ptr);
va += PageSize;
if (ptr != null)
{
ptr += PageSize;
}
}
}
private void SetPtEntry(long position, byte* ptr)
{
if (!IsValidPosition(position))
{
throw new ArgumentOutOfRangeException(nameof(position));
}
((byte**)_pageTable)[position >> PageBits] = ptr;
}
private void SetPtEntryFlag(long position, long flag)
{
ModifyPtEntryFlag(position, flag, setFlag: true);
}
private void ClearPtEntryFlag(long position, long flag)
{
ModifyPtEntryFlag(position, flag, setFlag: false);
}
private void ModifyPtEntryFlag(long position, long flag, bool setFlag)
{
IntPtr* pt = (IntPtr*)_pageTable;
while (true)
{
IntPtr old = pt[position >> PageBits];
IntPtr modified = setFlag
? new IntPtr(old.ToInt64() | flag)
: new IntPtr(old.ToInt64() & ~flag);
IntPtr origValue = Interlocked.CompareExchange(ref pt[position >> PageBits], modified, old);
if (origValue == old)
{
break;
}
}
}
public bool IsRegionModified(long position, long size)
{
if (!HasWriteWatchSupport)
{
return IsRegionModifiedFallback(position, size);
}
IntPtr address = Translate(position);
IntPtr baseAddr = address;
IntPtr expectedAddr = address;
long pendingPages = 0;
long pages = size / PageSize;
bool modified = false;
bool IsAnyPageModified()
{
IntPtr pendingSize = new IntPtr(pendingPages * PageSize);
IntPtr[] addresses = new IntPtr[pendingPages];
bool result = GetModifiedPages(baseAddr, pendingSize, addresses, out ulong count);
if (result)
{
return count != 0;
}
else
{
return true;
}
}
while (pages-- > 0)
{
if (address != expectedAddr)
{
modified |= IsAnyPageModified();
baseAddr = address;
pendingPages = 0;
}
expectedAddr = address + PageSize;
pendingPages++;
if (pages == 0)
{
break;
}
position += PageSize;
address = Translate(position);
}
if (pendingPages != 0)
{
modified |= IsAnyPageModified();
}
return modified;
}
private unsafe bool IsRegionModifiedFallback(long position, long size)
{
long endAddr = (position + size + PageMask) & ~PageMask;
bool modified = false;
while ((ulong)position < (ulong)endAddr)
{
if (IsValidPosition(position))
{
byte* ptr = ((byte**)_pageTable)[position >> PageBits];
ulong ptrUlong = (ulong)ptr;
if ((ptrUlong & PteFlagNotModified) == 0)
{
modified = true;
SetPtEntryFlag(position, PteFlagNotModified);
}
}
else
{
modified = true;
}
position += PageSize;
}
return modified;
}
public bool TryGetHostAddress(long position, long size, out IntPtr ptr)
{
if (IsContiguous(position, size))
{
ptr = (IntPtr)Translate(position);
return true;
}
ptr = IntPtr.Zero;
return false;
}
private bool IsContiguous(long position, long size)
{
long endPos = position + size;
position &= ~PageMask;
long expectedPa = GetPhysicalAddress(position);
while ((ulong)position < (ulong)endPos)
{
long pa = GetPhysicalAddress(position);
if (pa != expectedPa)
{
return false;
}
position += PageSize;
expectedPa += PageSize;
}
return true;
}
public bool IsValidPosition(long position)
{
return (ulong)position <= (ulong)AddressSpaceSize;
}
internal bool AtomicCompareExchange2xInt32(
@ -86,7 +349,7 @@ namespace ChocolArm64.Memory
AbortWithAlignmentFault(position);
}
IntPtr ptr = new IntPtr(TranslateWrite(position));
IntPtr ptr = TranslateWrite(position);
return InterlockedCompareExchange128(ptr, expectedLow, expectedHigh, desiredLow, desiredHigh);
}
@ -98,7 +361,7 @@ namespace ChocolArm64.Memory
AbortWithAlignmentFault(position);
}
IntPtr ptr = new IntPtr(Translate(position));
IntPtr ptr = Translate(position);
InterlockedRead128(ptr, out ulong low, out ulong high);
@ -371,7 +634,7 @@ namespace ChocolArm64.Memory
int copySize = (int)(pageLimit - position);
Marshal.Copy((IntPtr)Translate(position), data, offset, copySize);
Marshal.Copy(Translate(position), data, offset, copySize);
position += copySize;
offset += copySize;
@ -408,7 +671,7 @@ namespace ChocolArm64.Memory
int copySize = (int)(pageLimit - position);
Marshal.Copy((IntPtr)Translate(position), data, offset, copySize);
Marshal.Copy(Translate(position), data, offset, copySize);
position += copySize;
offset += copySize;
@ -571,7 +834,7 @@ namespace ChocolArm64.Memory
int copySize = (int)(pageLimit - position);
Marshal.Copy(data, offset, (IntPtr)TranslateWrite(position), copySize);
Marshal.Copy(data, offset, TranslateWrite(position), copySize);
position += copySize;
offset += copySize;
@ -601,7 +864,7 @@ namespace ChocolArm64.Memory
int copySize = (int)(pageLimit - position);
Marshal.Copy(data, offset, (IntPtr)TranslateWrite(position), copySize);
Marshal.Copy(data, offset, Translate(position), copySize);
position += copySize;
offset += copySize;
@ -614,8 +877,8 @@ namespace ChocolArm64.Memory
if (IsContiguous(src, size) &&
IsContiguous(dst, size))
{
byte* srcPtr = Translate(src);
byte* dstPtr = TranslateWrite(dst);
byte* srcPtr = (byte*)Translate(src);
byte* dstPtr = (byte*)Translate(dst);
Buffer.MemoryCopy(srcPtr, dstPtr, size, size);
}
@ -625,266 +888,6 @@ namespace ChocolArm64.Memory
}
}
public void Map(long va, long pa, long size)
{
SetPtEntries(va, _ramPtr + pa, size);
}
public void Unmap(long position, long size)
{
SetPtEntries(position, null, size);
StopObservingRegion(position, size);
}
public bool IsMapped(long position)
{
if (!(IsValidPosition(position)))
{
return false;
}
long l0 = (position >> PtLvl0Bit) & PtLvl0Mask;
long l1 = (position >> PtLvl1Bit) & PtLvl1Mask;
if (_pageTable[l0] == null)
{
return false;
}
return _pageTable[l0][l1] != null || _observedPages.ContainsKey(position >> PageBits);
}
public long GetPhysicalAddress(long virtualAddress)
{
byte* ptr = Translate(virtualAddress);
return (long)(ptr - _ramPtr);
}
internal byte* Translate(long position)
{
long l0 = (position >> PtLvl0Bit) & PtLvl0Mask;
long l1 = (position >> PtLvl1Bit) & PtLvl1Mask;
long old = position;
byte** lvl1 = _pageTable[l0];
if ((position >> (PtLvl0Bit + PtLvl0Bits)) != 0)
{
goto Unmapped;
}
if (lvl1 == null)
{
goto Unmapped;
}
position &= PageMask;
byte* ptr = lvl1[l1];
if (ptr == null)
{
goto Unmapped;
}
return ptr + position;
Unmapped:
return HandleNullPte(old);
}
private byte* HandleNullPte(long position)
{
long key = position >> PageBits;
if (_observedPages.TryGetValue(key, out IntPtr ptr))
{
return (byte*)ptr + (position & PageMask);
}
InvalidAccess?.Invoke(this, new MemoryAccessEventArgs(position));
throw new VmmPageFaultException(position);
}
internal byte* TranslateWrite(long position)
{
long l0 = (position >> PtLvl0Bit) & PtLvl0Mask;
long l1 = (position >> PtLvl1Bit) & PtLvl1Mask;
long old = position;
byte** lvl1 = _pageTable[l0];
if ((position >> (PtLvl0Bit + PtLvl0Bits)) != 0)
{
goto Unmapped;
}
if (lvl1 == null)
{
goto Unmapped;
}
position &= PageMask;
byte* ptr = lvl1[l1];
if (ptr == null)
{
goto Unmapped;
}
return ptr + position;
Unmapped:
return HandleNullPteWrite(old);
}
private byte* HandleNullPteWrite(long position)
{
long key = position >> PageBits;
MemoryAccessEventArgs e = new MemoryAccessEventArgs(position);
if (_observedPages.TryGetValue(key, out IntPtr ptr))
{
SetPtEntry(position, (byte*)ptr);
ObservedAccess?.Invoke(this, e);
return (byte*)ptr + (position & PageMask);
}
InvalidAccess?.Invoke(this, e);
throw new VmmPageFaultException(position);
}
private void SetPtEntries(long va, byte* ptr, long size)
{
long endPosition = (va + size + PageMask) & ~PageMask;
while ((ulong)va < (ulong)endPosition)
{
SetPtEntry(va, ptr);
va += PageSize;
if (ptr != null)
{
ptr += PageSize;
}
}
}
private void SetPtEntry(long position, byte* ptr)
{
if (!IsValidPosition(position))
{
throw new ArgumentOutOfRangeException(nameof(position));
}
long l0 = (position >> PtLvl0Bit) & PtLvl0Mask;
long l1 = (position >> PtLvl1Bit) & PtLvl1Mask;
if (_pageTable[l0] == null)
{
byte** lvl1 = (byte**)Marshal.AllocHGlobal(PtLvl1Size * IntPtr.Size);
for (int zl1 = 0; zl1 < PtLvl1Size; zl1++)
{
lvl1[zl1] = null;
}
Thread.MemoryBarrier();
_pageTable[l0] = lvl1;
}
_pageTable[l0][l1] = ptr;
}
public void StartObservingRegion(long position, long size)
{
long endPosition = (position + size + PageMask) & ~PageMask;
position &= ~PageMask;
while ((ulong)position < (ulong)endPosition)
{
_observedPages[position >> PageBits] = (IntPtr)Translate(position);
SetPtEntry(position, null);
position += PageSize;
}
}
public void StopObservingRegion(long position, long size)
{
long endPosition = (position + size + PageMask) & ~PageMask;
while (position < endPosition)
{
lock (_observedPages)
{
if (_observedPages.TryRemove(position >> PageBits, out IntPtr ptr))
{
SetPtEntry(position, (byte*)ptr);
}
}
position += PageSize;
}
}
public bool TryGetHostAddress(long position, long size, out IntPtr ptr)
{
if (IsContiguous(position, size))
{
ptr = (IntPtr)Translate(position);
return true;
}
ptr = IntPtr.Zero;
return false;
}
private bool IsContiguous(long position, long size)
{
long endPos = position + size;
position &= ~PageMask;
long expectedPa = GetPhysicalAddress(position);
while ((ulong)position < (ulong)endPos)
{
long pa = GetPhysicalAddress(position);
if (pa != expectedPa)
{
return false;
}
position += PageSize;
expectedPa += PageSize;
}
return true;
}
public bool IsValidPosition(long position)
{
return position >> (PtLvl0Bits + PtLvl1Bits + PageBits) == 0;
}
public void Dispose()
{
Dispose(true);
@ -892,24 +895,12 @@ Unmapped:
protected virtual void Dispose(bool disposing)
{
if (_pageTable == null)
IntPtr ptr = Interlocked.Exchange(ref _pageTable, IntPtr.Zero);
if (ptr != IntPtr.Zero)
{
return;
Free(ptr);
}
for (int l0 = 0; l0 < PtLvl0Size; l0++)
{
if (_pageTable[l0] != null)
{
Marshal.FreeHGlobal((IntPtr)_pageTable[l0]);
}
_pageTable[l0] = null;
}
Marshal.FreeHGlobal((IntPtr)_pageTable);
_pageTable = null;
}
}
}

67
ChocolArm64/Translation/ILEmitterCtx.cs
View file

@ -1,5 +1,6 @@
using ChocolArm64.Decoders;
using ChocolArm64.Instructions;
using ChocolArm64.Memory;
using ChocolArm64.State;
using System;
using System.Collections.Generic;
@ -10,6 +11,8 @@ namespace ChocolArm64.Translation
{
class ILEmitterCtx
{
public MemoryManager Memory { get; }
private TranslatorCache _cache;
private TranslatorQueue _queue;
@ -43,19 +46,35 @@ namespace ChocolArm64.Translation
//values needed by some functions, since IL doesn't have a swap instruction.
//You can use any value here as long it doesn't conflict with the indices
//for the other registers. Any value >= 64 or < 0 will do.
private const int IntTmpIndex = -1;
private const int RorTmpIndex = -2;
private const int CmpOptTmp1Index = -3;
private const int CmpOptTmp2Index = -4;
private const int VecTmp1Index = -5;
private const int VecTmp2Index = -6;
private const int IntTmp2Index = -7;
private const int ReservedLocalsCount = 64;
public ILEmitterCtx(TranslatorCache cache, TranslatorQueue queue, TranslationTier tier, Block graph)
private const int RorTmpIndex = ReservedLocalsCount + 0;
private const int CmpOptTmp1Index = ReservedLocalsCount + 1;
private const int CmpOptTmp2Index = ReservedLocalsCount + 2;
private const int IntGpTmp1Index = ReservedLocalsCount + 3;
private const int IntGpTmp2Index = ReservedLocalsCount + 4;
private const int UserIntTempStart = ReservedLocalsCount + 5;
//Vectors are part of another "set" of locals.
private const int VecGpTmp1Index = ReservedLocalsCount + 0;
private const int VecGpTmp2Index = ReservedLocalsCount + 1;
private const int UserVecTempStart = ReservedLocalsCount + 2;
private static int _userIntTempCount;
private static int _userVecTempCount;
public ILEmitterCtx(
MemoryManager memory,
TranslatorCache cache,
TranslatorQueue queue,
TranslationTier tier,
Block graph)
{
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_currBlock = graph ?? throw new ArgumentNullException(nameof(graph));
Memory = memory ?? throw new ArgumentNullException(nameof(memory));
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_currBlock = graph ?? throw new ArgumentNullException(nameof(graph));
Tier = tier;
@ -76,6 +95,16 @@ namespace ChocolArm64.Translation
AdvanceOpCode();
}
public static int GetIntTempIndex()
{
return UserIntTempStart + _userIntTempCount++;
}
public static int GetVecTempIndex()
{
return UserVecTempStart + _userVecTempCount++;
}
public ILBlock[] GetILBlocks()
{
EmitAllOpCodes();
@ -560,17 +589,17 @@ namespace ChocolArm64.Translation
_ilBlock.Add(new ILOpCodeStoreState(_ilBlock));
}
public void EmitLdtmp() => EmitLdint(IntTmpIndex);
public void EmitSttmp() => EmitStint(IntTmpIndex);
public void EmitLdtmp() => EmitLdint(IntGpTmp1Index);
public void EmitSttmp() => EmitStint(IntGpTmp1Index);
public void EmitLdtmp2() => EmitLdint(IntTmp2Index);
public void EmitSttmp2() => EmitStint(IntTmp2Index);
public void EmitLdtmp2() => EmitLdint(IntGpTmp2Index);
public void EmitSttmp2() => EmitStint(IntGpTmp2Index);
public void EmitLdvectmp() => EmitLdvec(VecTmp1Index);
public void EmitStvectmp() => EmitStvec(VecTmp1Index);
public void EmitLdvectmp() => EmitLdvec(VecGpTmp1Index);
public void EmitStvectmp() => EmitStvec(VecGpTmp1Index);
public void EmitLdvectmp2() => EmitLdvec(VecTmp2Index);
public void EmitStvectmp2() => EmitStvec(VecTmp2Index);
public void EmitLdvectmp2() => EmitLdvec(VecGpTmp2Index);
public void EmitStvectmp2() => EmitStvec(VecGpTmp2Index);
public void EmitLdint(int index) => Ldloc(index, IoType.Int);
public void EmitStint(int index) => Stloc(index, IoType.Int);

4
ChocolArm64/Translation/Translator.cs
View file

@ -138,7 +138,7 @@ namespace ChocolArm64.Translation
{
Block block = Decoder.DecodeBasicBlock(_memory, position, mode);
ILEmitterCtx context = new ILEmitterCtx(_cache, _queue, TranslationTier.Tier0, block);
ILEmitterCtx context = new ILEmitterCtx(_memory, _cache, _queue, TranslationTier.Tier0, block);
string subName = GetSubroutineName(position);
@ -153,7 +153,7 @@ namespace ChocolArm64.Translation
{
Block graph = Decoder.DecodeSubroutine(_memory, position, mode);
ILEmitterCtx context = new ILEmitterCtx(_cache, _queue, TranslationTier.Tier1, graph);
ILEmitterCtx context = new ILEmitterCtx(_memory, _cache, _queue, TranslationTier.Tier1, graph);
ILBlock[] ilBlocks = context.GetILBlocks();

34
Ryujinx.Graphics/Memory/NvGpuVmmCache.cs
View file

@ -1,4 +1,3 @@
using ChocolArm64.Events;
using ChocolArm64.Memory;
using System.Collections.Concurrent;
@ -19,35 +18,28 @@ namespace Ryujinx.Graphics.Memory
{
_memory = memory;
_memory.ObservedAccess += MemoryAccessHandler;
CachedPages = new ConcurrentDictionary<long, int>[1 << 20];
}
private void MemoryAccessHandler(object sender, MemoryAccessEventArgs e)
{
long pa = _memory.GetPhysicalAddress(e.Position);
CachedPages[pa >> PageBits]?.Clear();
}
public bool IsRegionModified(long position, long size, NvGpuBufferType bufferType)
{
long pa = _memory.GetPhysicalAddress(position);
long va = position;
long addr = pa;
long pa = _memory.GetPhysicalAddress(va);
long endAddr = (addr + size + PageMask) & ~PageMask;
long endAddr = (va + size + PageMask) & ~PageMask;
long addrTruncated = va & ~PageMask;
bool modified = _memory.IsRegionModified(addrTruncated, endAddr - addrTruncated);
int newBuffMask = 1 << (int)bufferType;
_memory.StartObservingRegion(position, size);
long cachedPagesCount = 0;
while (addr < endAddr)
while (va < endAddr)
{
long page = addr >> PageBits;
long page = _memory.GetPhysicalAddress(va) >> PageBits;
ConcurrentDictionary<long, int> dictionary = CachedPages[page];
@ -57,6 +49,10 @@ namespace Ryujinx.Graphics.Memory
CachedPages[page] = dictionary;
}
else if (modified)
{
CachedPages[page].Clear();
}
if (dictionary.TryGetValue(pa, out int currBuffMask))
{
@ -74,10 +70,10 @@ namespace Ryujinx.Graphics.Memory
dictionary[pa] = newBuffMask;
}
addr += PageSize;
va += PageSize;
}
return cachedPagesCount != (endAddr - pa + PageMask) >> PageBits;
return cachedPagesCount != (endAddr - addrTruncated) >> PageBits;
}
}
}

7
Ryujinx.HLE/DeviceMemory.cs
View file

@ -1,3 +1,4 @@
using ChocolArm64.Memory;
using System;
using System.Runtime.InteropServices;
@ -7,13 +8,13 @@ namespace Ryujinx.HLE
{
public const long RamSize = 4L * 1024 * 1024 * 1024;
public IntPtr RamPointer { get; private set; }
public IntPtr RamPointer { get; }
private unsafe byte* _ramPtr;
public unsafe DeviceMemory()
{
RamPointer = Marshal.AllocHGlobal(new IntPtr(RamSize));
RamPointer = MemoryAlloc.AllocateWriteTracked(RamSize);
_ramPtr = (byte*)RamPointer;
}
@ -177,7 +178,7 @@ namespace Ryujinx.HLE
protected virtual void Dispose(bool disposing)
{
Marshal.FreeHGlobal(RamPointer);
MemoryAlloc.Free(RamPointer);
}
}
}

7
Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs
View file

@ -95,8 +95,6 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
CpuMemory = new MemoryManager(system.Device.Memory.RamPointer);
CpuMemory.InvalidAccess += InvalidAccessHandler;
AddressArbiter = new KAddressArbiter(system);
MemoryManager = new KMemoryManager(system, CpuMemory);
@ -1010,11 +1008,6 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
}
}
private void InvalidAccessHandler(object sender, MemoryAccessEventArgs e)
{
PrintCurrentThreadStackTrace();
}
public void PrintCurrentThreadStackTrace()
{
System.Scheduler.GetCurrentThread().PrintGuestStackTrace();

5
Ryujinx.HLE/HOS/Kernel/SupervisorCall/SvcMemory.cs
View file

@ -62,11 +62,6 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
attributeMask,
attributeValue);
if (result == KernelResult.Success)
{
_memory.StopObservingRegion((long)position, (long)size);
}
return result;
}

2
Ryujinx.Tests/Cpu/CpuTest.cs
View file

@ -50,7 +50,7 @@ namespace Ryujinx.Tests.Cpu
_entryPoint = Position;
_ramPointer = Marshal.AllocHGlobal(new IntPtr(_size));
_memory = new MemoryManager(_ramPointer);
_memory = new MemoryManager(_ramPointer, 1L << 32);
_memory.Map(Position, 0, _size);
Translator translator = new Translator(_memory);